1# -*- coding: utf-8 -*- #
2# Copyright 2018 Google LLC. All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#    http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16"""Wrapper module for ensuring consistent usage of yaml parsing.
17
18This module forces parsing to use version 1.1 of the YAML spec if not
19otherwise specified by the loading method arguments.
20However, dumping uses version 1.2.
21It also prevents use of unsafe loading and dumping.
22"""
23
24from __future__ import absolute_import
25from __future__ import division
26from __future__ import unicode_literals
27
28import collections
29
30from googlecloudsdk.core import exceptions
31from googlecloudsdk.core import yaml_location_value
32from googlecloudsdk.core.util import files
33
34from ruamel import yaml
35import six
36
37try:
38  # Python 3.3 and above.
39  collections_abc = collections.abc
40except AttributeError:
41  collections_abc = collections
42
43
44VERSION_1_1 = '1.1'
45VERSION_1_2 = '1.2'
46
47
48# YAML unfortunately uses a bunch of global class state for this kind of stuff.
49# We don't have to do it at import but the other option would be to do it every
50# time we try to dump something (which is worse for performance that just
51# doing it once). This allows OrderedDicts to be serialized as if they were
52# normal dicts.
53yaml.add_representer(
54    collections.OrderedDict,
55    yaml.dumper.SafeRepresenter.represent_dict,
56    Dumper=yaml.dumper.SafeDumper)
57yaml.add_representer(
58    collections.OrderedDict,
59    yaml.dumper.RoundTripRepresenter.represent_dict,
60    Dumper=yaml.dumper.RoundTripDumper)
61
62
63# Always output None as "null", instead of just empty.
64yaml.add_representer(
65    type(None),
66    lambda self, _: self.represent_scalar('tag:yaml.org,2002:null', 'null'),
67    Dumper=yaml.dumper.RoundTripDumper)
68
69
70class Error(exceptions.Error):
71  """Top level error for this module.
72
73  Attributes:
74    inner_error: Exception, The original exception that is being wrapped. This
75      will always be populated.
76    file: str, The path to the thing being loaded (if applicable). This is not
77      necessarily a literal file (it could be a URL or any hint the calling
78      code passes in). It should only be used for more descriptive error
79      messages.
80  """
81
82  def __init__(self, e, verb, f=None):
83    file_text = ' from [{}]'.format(f) if f else ''
84    super(Error, self).__init__(
85        'Failed to {} YAML{}: {}'.format(verb, file_text, e))
86    self.inner_error = e
87    self.file = f
88
89
90class YAMLParseError(Error):
91  """An error that wraps all YAML parsing errors."""
92
93  def __init__(self, e, f=None):
94    super(YAMLParseError, self).__init__(e, verb='parse', f=f)
95
96
97class FileLoadError(Error):
98  """An error that wraps errors when loading/reading files."""
99
100  def __init__(self, e, f):
101    super(FileLoadError, self).__init__(e, verb='load', f=f)
102
103
104def load(stream,
105         file_hint=None,
106         round_trip=False,
107         location_value=False,
108         version=VERSION_1_1):
109  """Loads YAML from the given steam.
110
111  Args:
112    stream: A file like object or string that can be read from.
113    file_hint: str, The name of a file or url that the stream data is coming
114      from. This is used for better error handling. If you have the actual file,
115      you should use load_file() instead. Sometimes the file cannot be read
116      directly so you can use a stream here and hint as to where the data is
117      coming from.
118    round_trip: bool, True to use the RoundTripLoader which preserves ordering
119      and line numbers.
120    location_value: bool, True to use a loader that preserves ordering and line
121      numbers for all values. Each YAML data item is an object with value and
122      lc attributes, where lc.line and lc.col are the line and column location
123      for the item in the YAML source file.
124    version: str, YAML version to use when parsing.
125
126  Raises:
127    YAMLParseError: If the data could not be parsed.
128
129  Returns:
130    The parsed YAML data.
131  """
132  try:
133    if location_value:
134      return yaml_location_value.LocationValueLoad(stream)
135    loader = yaml.RoundTripLoader if round_trip else yaml.SafeLoader
136    return yaml.load(stream, loader, version=version)
137  except yaml.YAMLError as e:
138    raise YAMLParseError(e, f=file_hint)
139
140
141def load_all(stream, file_hint=None, version=VERSION_1_1, round_trip=False):
142  """Loads multiple YAML documents from the given steam.
143
144  Args:
145    stream: A file like object or string that can be read from.
146    file_hint: str, The name of a file or url that the stream data is coming
147      from. See load() for more information.
148    version: str, YAML version to use when parsing.
149    round_trip: bool, True to use the RoundTripLoader which preserves ordering
150      and line numbers.
151
152  Raises:
153    YAMLParseError: If the data could not be parsed.
154
155  Yields:
156    The parsed YAML data.
157  """
158  loader = yaml.RoundTripLoader if round_trip else yaml.SafeLoader
159  try:
160    for x in yaml.load_all(stream, loader, version=version):
161      yield x
162  except yaml.YAMLError as e:
163    raise YAMLParseError(e, f=file_hint)
164
165
166def load_path(path,
167              round_trip=False,
168              location_value=False,
169              version=VERSION_1_1):
170  """Loads YAML from the given file path.
171
172  Args:
173    path: str, A file path to open and read from.
174    round_trip: bool, True to use the RoundTripLoader which preserves ordering
175      and line numbers.
176    location_value: bool, True to use a loader that preserves ordering and line
177      numbers for all values. Each YAML data item is an object with value and
178      lc attributes, where lc.line and lc.col are the line and column location
179      for the item in the YAML source file.
180    version: str, YAML version to use when parsing.
181
182  Raises:
183    YAMLParseError: If the data could not be parsed.
184    FileLoadError: If the file could not be opened or read.
185
186  Returns:
187    The parsed YAML data.
188  """
189  try:
190    with files.FileReader(path) as fp:
191      return load(
192          fp,
193          file_hint=path,
194          round_trip=round_trip,
195          location_value=location_value,
196          version=version)
197  except files.Error as e:
198    raise FileLoadError(e, f=path)
199
200
201def load_all_path(path, version=VERSION_1_1, round_trip=False):
202  """Loads multiple YAML documents from the given file path.
203
204  Args:
205    path: str, A file path to open and read from.
206    version: str, YAML version to use when parsing.
207    round_trip: bool, True to use the RoundTripLoader which preserves ordering
208      and line numbers.
209
210  Raises:
211    YAMLParseError: If the data could not be parsed.
212    FileLoadError: If the file could not be opened or read.
213
214  Yields:
215    The parsed YAML data.
216  """
217  try:
218    with files.FileReader(path) as fp:
219      for x in load_all(fp,
220                        file_hint=path,
221                        version=version,
222                        round_trip=round_trip):
223        yield x
224  except files.Error as e:
225    # EnvironmentError is parent of IOError, OSError and WindowsError.
226    # Raised when file does not exist or can't be opened/read.
227    raise FileLoadError(e, f=path)
228
229
230def dump(data, stream=None, round_trip=False, **kwargs):
231  """Dumps the given YAML data to the stream.
232
233  Args:
234    data: The YAML serializable Python object to dump.
235    stream: The stream to write the data to or None to return it as a string.
236    round_trip: bool, True to use the RoundTripDumper which preserves ordering
237      and line numbers if the yaml was loaded in round trip mode.
238    **kwargs: Other arguments to the dump method.
239
240  Returns:
241    The string representation of the YAML data if stream is None.
242  """
243  method = yaml.round_trip_dump if round_trip else yaml.safe_dump
244  return method(data, stream=stream, default_flow_style=False, indent=2,
245                **kwargs)
246
247
248def dump_all(documents, stream=None, **kwargs):
249  """Dumps multiple YAML documents to the stream.
250
251  Args:
252    documents: An iterable of YAML serializable Python objects to dump.
253    stream: The stream to write the data to or None to return it as a string.
254    **kwargs: Other arguments to the dump method.
255
256  Returns:
257    The string representation of the YAML data if stream is None.
258  """
259  return yaml.safe_dump_all(
260      documents, stream=stream, default_flow_style=False, indent=2, **kwargs)
261
262
263def dump_all_round_trip(documents, stream=None, **kwargs):
264  """Dumps multiple YAML documents to the stream using the RoundTripDumper.
265
266  Args:
267    documents: An iterable of YAML serializable Python objects to dump.
268    stream: The stream to write the data to or None to return it as a string.
269    **kwargs: Other arguments to the dump method.
270
271  Returns:
272    The string representation of the YAML data if stream is None.
273  """
274  return yaml.dump_all(
275      documents, stream=stream, default_flow_style=False, indent=2,
276      Dumper=yaml.RoundTripDumper, **kwargs)
277
278
279def convert_to_block_text(data):
280  r"""This processes the given dict or list so it will render as block text.
281
282  By default, the yaml dumper will write multiline strings out as a double
283  quoted string that just includes '\n'. Calling this on the data strucuture
284  will make it use the '|-' notation.
285
286  Args:
287    data: {} or [], The data structure to process.
288  """
289  yaml.scalarstring.walk_tree(data)
290
291
292def list_like(item):
293  """Return True if the item is like a list: a MutableSequence."""
294  return isinstance(item, collections_abc.MutableSequence)
295
296
297def dict_like(item):
298  """Return True if the item is like a dict: a MutableMapping."""
299  return isinstance(item, collections_abc.MutableMapping)
300
301
302def strip_locations(obj):
303  if list_like(obj):
304    return [strip_locations(item) for item in obj]
305  if dict_like(obj):
306    return {key: strip_locations(value) for key, value in six.iteritems(obj)}
307  return obj.value
308