1# Copyright 2014 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Helper functions for dealing with Cloud Datastore's Protobuf API. 16 17The non-private functions are part of the API. 18""" 19 20import datetime 21import itertools 22 23from google.protobuf import struct_pb2 24from google.type import latlng_pb2 25from proto.datetime_helpers import DatetimeWithNanoseconds 26 27from google.cloud._helpers import _datetime_to_pb_timestamp 28from google.cloud.datastore_v1.types import datastore as datastore_pb2 29from google.cloud.datastore_v1.types import entity as entity_pb2 30from google.cloud.datastore.entity import Entity 31from google.cloud.datastore.key import Key 32 33 34def _get_meaning(value_pb, is_list=False): 35 """Get the meaning from a protobuf value. 36 37 :type value_pb: :class:`.entity_pb2.Value._pb` 38 :param value_pb: The *raw* protobuf value to be checked for an 39 associated meaning. 40 41 :type is_list: bool 42 :param is_list: Boolean indicating if the ``value_pb`` contains 43 a list value. 44 45 :rtype: int 46 :returns: The meaning for the ``value_pb`` if one is set, else 47 :data:`None`. For a list value, if there are disagreeing 48 means it just returns a list of meanings. If all the 49 list meanings agree, it just condenses them. 50 """ 51 if is_list: 52 53 values = value_pb.array_value.values 54 55 # An empty list will have no values, hence no shared meaning 56 # set among them. 57 if len(values) == 0: 58 return None 59 60 # We check among all the meanings, some of which may be None, 61 # the rest which may be enum/int values. 62 all_meanings = [_get_meaning(sub_value_pb) for sub_value_pb in values] 63 unique_meanings = set(all_meanings) 64 65 if len(unique_meanings) == 1: 66 # If there is a unique meaning, we preserve it. 67 return unique_meanings.pop() 68 else: # We know len(value_pb.array_value.values) > 0. 69 # If the meaning is not unique, just return all of them. 70 return all_meanings 71 72 elif value_pb.meaning: # Simple field (int32). 73 return value_pb.meaning 74 75 return None 76 77 78def _new_value_pb(entity_pb, name): 79 """Add (by name) a new ``Value`` protobuf to an entity protobuf. 80 81 :type entity_pb: :class:`.entity_pb2.Entity` 82 :param entity_pb: An entity protobuf to add a new property to. 83 84 :type name: str 85 :param name: The name of the new property. 86 87 :rtype: :class:`.entity_pb2.Value` 88 :returns: The new ``Value`` protobuf that was added to the entity. 89 """ 90 # TODO(microgenerator): shouldn't need this. the issue is that 91 # we have wrapped and non-wrapped protos coming here. 92 properties = getattr(entity_pb.properties, "_pb", entity_pb.properties) 93 return properties.get_or_create(name) 94 95 96def entity_from_protobuf(pb): 97 """Factory method for creating an entity based on a protobuf. 98 99 The protobuf should be one returned from the Cloud Datastore 100 Protobuf API. 101 102 :type pb: :class:`.entity_pb2.Entity` 103 :param pb: The Protobuf representing the entity. 104 105 :rtype: :class:`google.cloud.datastore.entity.Entity` 106 :returns: The entity derived from the protobuf. 107 """ 108 if isinstance(pb, entity_pb2.Entity): 109 pb = pb._pb 110 111 key = None 112 if pb.HasField("key"): # Message field (Key) 113 key = key_from_protobuf(pb.key) 114 115 entity_props = {} 116 entity_meanings = {} 117 exclude_from_indexes = [] 118 119 for prop_name, value_pb in pb.properties.items(): 120 value = _get_value_from_value_pb(value_pb) 121 entity_props[prop_name] = value 122 123 # Check if the property has an associated meaning. 124 is_list = isinstance(value, list) 125 meaning = _get_meaning(value_pb, is_list=is_list) 126 if meaning is not None: 127 entity_meanings[prop_name] = (meaning, value) 128 129 # Check if ``value_pb`` was excluded from index. Lists need to be 130 # special-cased and we require all ``exclude_from_indexes`` values 131 # in a list agree. 132 if is_list and len(value) > 0: 133 exclude_values = set( 134 value_pb.exclude_from_indexes 135 for value_pb in value_pb.array_value.values 136 ) 137 if len(exclude_values) != 1: 138 raise ValueError( 139 "For an array_value, subvalues must either " 140 "all be indexed or all excluded from " 141 "indexes." 142 ) 143 144 if exclude_values.pop(): 145 exclude_from_indexes.append(prop_name) 146 else: 147 if value_pb.exclude_from_indexes: 148 exclude_from_indexes.append(prop_name) 149 150 entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes) 151 entity.update(entity_props) 152 entity._meanings.update(entity_meanings) 153 return entity 154 155 156def _set_pb_meaning_from_entity(entity, name, value, value_pb, is_list=False): 157 """Add meaning information (from an entity) to a protobuf. 158 159 :type entity: :class:`google.cloud.datastore.entity.Entity` 160 :param entity: The entity to be turned into a protobuf. 161 162 :type name: str 163 :param name: The name of the property. 164 165 :type value: object 166 :param value: The current value stored as property ``name``. 167 168 :type value_pb: :class:`.entity_pb2.Value` 169 :param value_pb: The protobuf value to add meaning / meanings to. 170 171 :type is_list: bool 172 :param is_list: (Optional) Boolean indicating if the ``value`` is 173 a list value. 174 """ 175 if name not in entity._meanings: 176 return 177 178 meaning, orig_value = entity._meanings[name] 179 # Only add the meaning back to the protobuf if the value is 180 # unchanged from when it was originally read from the API. 181 if orig_value is not value: 182 return 183 184 # For lists, we set meaning on each sub-element. 185 if is_list: 186 if not isinstance(meaning, list): 187 meaning = itertools.repeat(meaning) 188 val_iter = zip(value_pb.array_value.values, meaning) 189 for sub_value_pb, sub_meaning in val_iter: 190 if sub_meaning is not None: 191 sub_value_pb.meaning = sub_meaning 192 else: 193 value_pb.meaning = meaning 194 195 196def entity_to_protobuf(entity): 197 """Converts an entity into a protobuf. 198 199 :type entity: :class:`google.cloud.datastore.entity.Entity` 200 :param entity: The entity to be turned into a protobuf. 201 202 :rtype: :class:`.entity_pb2.Entity` 203 :returns: The protobuf representing the entity. 204 """ 205 entity_pb = entity_pb2.Entity() 206 if entity.key is not None: 207 key_pb = entity.key.to_protobuf() 208 entity_pb._pb.key.CopyFrom(key_pb._pb) 209 210 for name, value in entity.items(): 211 value_is_list = isinstance(value, list) 212 213 value_pb = _new_value_pb(entity_pb, name) 214 # Set the appropriate value. 215 _set_protobuf_value(value_pb, value) 216 217 # Add index information to protobuf. 218 if name in entity.exclude_from_indexes: 219 if not value_is_list: 220 value_pb.exclude_from_indexes = True 221 222 for sub_value in value_pb.array_value.values: 223 sub_value.exclude_from_indexes = True 224 225 # Add meaning information to protobuf. 226 _set_pb_meaning_from_entity( 227 entity, name, value, value_pb, is_list=value_is_list 228 ) 229 230 return entity_pb 231 232 233def get_read_options(eventual, transaction_id): 234 """Validate rules for read options, and assign to the request. 235 236 Helper method for ``lookup()`` and ``run_query``. 237 238 :type eventual: bool 239 :param eventual: Flag indicating if ``EVENTUAL`` or ``STRONG`` 240 consistency should be used. 241 242 :type transaction_id: bytes 243 :param transaction_id: A transaction identifier (may be null). 244 245 :rtype: :class:`.datastore_pb2.ReadOptions` 246 :returns: The read options corresponding to the inputs. 247 :raises: :class:`ValueError` if ``eventual`` is ``True`` and the 248 ``transaction_id`` is not ``None``. 249 """ 250 if transaction_id is None: 251 if eventual: 252 return datastore_pb2.ReadOptions( 253 read_consistency=datastore_pb2.ReadOptions.ReadConsistency.EVENTUAL 254 ) 255 else: 256 return datastore_pb2.ReadOptions() 257 else: 258 if eventual: 259 raise ValueError("eventual must be False when in a transaction") 260 else: 261 return datastore_pb2.ReadOptions(transaction=transaction_id) 262 263 264def key_from_protobuf(pb): 265 """Factory method for creating a key based on a protobuf. 266 267 The protobuf should be one returned from the Cloud Datastore 268 Protobuf API. 269 270 :type pb: :class:`.entity_pb2.Key` 271 :param pb: The Protobuf representing the key. 272 273 :rtype: :class:`google.cloud.datastore.key.Key` 274 :returns: a new `Key` instance 275 """ 276 path_args = [] 277 for element in pb.path: 278 path_args.append(element.kind) 279 if element.id: # Simple field (int64) 280 path_args.append(element.id) 281 # This is safe: we expect proto objects returned will only have 282 # one of `name` or `id` set. 283 if element.name: # Simple field (string) 284 path_args.append(element.name) 285 286 project = None 287 if pb.partition_id.project_id: # Simple field (string) 288 project = pb.partition_id.project_id 289 namespace = None 290 if pb.partition_id.namespace_id: # Simple field (string) 291 namespace = pb.partition_id.namespace_id 292 293 return Key(*path_args, namespace=namespace, project=project) 294 295 296def _pb_attr_value(val): 297 """Given a value, return the protobuf attribute name and proper value. 298 299 The Protobuf API uses different attribute names based on value types 300 rather than inferring the type. This function simply determines the 301 proper attribute name based on the type of the value provided and 302 returns the attribute name as well as a properly formatted value. 303 304 Certain value types need to be coerced into a different type (such 305 as a `datetime.datetime` into an integer timestamp, or a 306 `google.cloud.datastore.key.Key` into a Protobuf representation. This 307 function handles that for you. 308 309 .. note:: 310 Values which are "text" ('unicode' in Python2, 'str' in Python3) map 311 to 'string_value' in the datastore; values which are "bytes" 312 ('str' in Python2, 'bytes' in Python3) map to 'blob_value'. 313 314 For example: 315 316 .. testsetup:: pb-attr-value 317 318 from google.cloud.datastore.helpers import _pb_attr_value 319 320 .. doctest:: pb-attr-value 321 322 >>> _pb_attr_value(1234) 323 ('integer_value', 1234) 324 >>> _pb_attr_value('my_string') 325 ('string_value', 'my_string') 326 327 :type val: 328 :class:`datetime.datetime`, :class:`google.cloud.datastore.key.Key`, 329 bool, float, integer, bytes, str, unicode, 330 :class:`google.cloud.datastore.entity.Entity`, dict, list, 331 :class:`google.cloud.datastore.helpers.GeoPoint`, NoneType 332 :param val: The value to be scrutinized. 333 334 :rtype: tuple 335 :returns: A tuple of the attribute name and proper value type. 336 """ 337 338 if isinstance(val, datetime.datetime): 339 name = "timestamp" 340 value = _datetime_to_pb_timestamp(val) 341 elif isinstance(val, Key): 342 name, value = "key", val.to_protobuf() 343 elif isinstance(val, bool): 344 name, value = "boolean", val 345 elif isinstance(val, float): 346 name, value = "double", val 347 elif isinstance(val, int): 348 name, value = "integer", val 349 elif isinstance(val, str): 350 name, value = "string", val 351 elif isinstance(val, bytes): 352 name, value = "blob", val 353 elif isinstance(val, Entity): 354 name, value = "entity", val 355 elif isinstance(val, dict): 356 entity_val = Entity(key=None) 357 entity_val.update(val) 358 name, value = "entity", entity_val 359 elif isinstance(val, list): 360 name, value = "array", val 361 elif isinstance(val, GeoPoint): 362 name, value = "geo_point", val.to_protobuf() 363 elif val is None: 364 name, value = "null", struct_pb2.NULL_VALUE 365 else: 366 raise ValueError("Unknown protobuf attr type", type(val)) 367 368 return name + "_value", value 369 370 371def _get_value_from_value_pb(pb): 372 """Given a protobuf for a Value, get the correct value. 373 374 The Cloud Datastore Protobuf API returns a Property Protobuf which 375 has one value set and the rest blank. This function retrieves the 376 the one value provided. 377 378 Some work is done to coerce the return value into a more useful type 379 (particularly in the case of a timestamp value, or a key value). 380 381 :type pb: :class:`.entity_pb2.Value._pb` 382 :param pb: The *raw* Value Protobuf. 383 384 :rtype: object 385 :returns: The value provided by the Protobuf. 386 :raises: :class:`ValueError <exceptions.ValueError>` if no value type 387 has been set. 388 """ 389 value_type = pb.WhichOneof("value_type") 390 391 if value_type == "timestamp_value": 392 result = DatetimeWithNanoseconds.from_timestamp_pb(pb.timestamp_value) 393 394 elif value_type == "key_value": 395 result = key_from_protobuf(pb.key_value) 396 397 elif value_type == "boolean_value": 398 result = pb.boolean_value 399 400 elif value_type == "double_value": 401 result = pb.double_value 402 403 elif value_type == "integer_value": 404 result = pb.integer_value 405 406 elif value_type == "string_value": 407 result = pb.string_value 408 409 elif value_type == "blob_value": 410 result = pb.blob_value 411 412 elif value_type == "entity_value": 413 result = entity_from_protobuf(pb.entity_value) 414 415 elif value_type == "array_value": 416 result = [ 417 _get_value_from_value_pb(item_value) for item_value in pb.array_value.values 418 ] 419 420 elif value_type == "geo_point_value": 421 result = GeoPoint(pb.geo_point_value.latitude, pb.geo_point_value.longitude,) 422 423 elif value_type == "null_value": 424 result = None 425 426 else: 427 raise ValueError("Value protobuf did not have any value set") 428 429 return result 430 431 432def _set_protobuf_value(value_pb, val): 433 """Assign 'val' to the correct subfield of 'value_pb'. 434 435 The Protobuf API uses different attribute names based on value types 436 rather than inferring the type. 437 438 Some value types (entities, keys, lists) cannot be directly 439 assigned; this function handles them correctly. 440 441 :type value_pb: :class:`.entity_pb2.Value` 442 :param value_pb: The value protobuf to which the value is being assigned. 443 444 :type val: :class:`datetime.datetime`, boolean, float, integer, string, 445 :class:`google.cloud.datastore.key.Key`, 446 :class:`google.cloud.datastore.entity.Entity` 447 :param val: The value to be assigned. 448 """ 449 attr, val = _pb_attr_value(val) 450 if attr == "key_value": 451 value_pb.key_value.CopyFrom(val._pb) 452 elif attr == "timestamp_value": 453 value_pb.timestamp_value.CopyFrom(val) 454 elif attr == "entity_value": 455 entity_pb = entity_to_protobuf(val) 456 value_pb.entity_value.CopyFrom(entity_pb._pb) 457 elif attr == "array_value": 458 if len(val) == 0: 459 array_value = entity_pb2.ArrayValue(values=[])._pb 460 value_pb.array_value.CopyFrom(array_value) 461 else: 462 l_pb = value_pb.array_value.values 463 for item in val: 464 i_pb = l_pb.add() 465 _set_protobuf_value(i_pb, item) 466 elif attr == "geo_point_value": 467 value_pb.geo_point_value.CopyFrom(val) 468 else: # scalar, just assign 469 setattr(value_pb, attr, val) 470 471 472class GeoPoint(object): 473 """Simple container for a geo point value. 474 475 :type latitude: float 476 :param latitude: Latitude of a point. 477 478 :type longitude: float 479 :param longitude: Longitude of a point. 480 """ 481 482 def __init__(self, latitude, longitude): 483 self.latitude = latitude 484 self.longitude = longitude 485 486 def to_protobuf(self): 487 """Convert the current object to protobuf. 488 489 :rtype: :class:`google.type.latlng_pb2.LatLng`. 490 :returns: The current point as a protobuf. 491 """ 492 return latlng_pb2.LatLng(latitude=self.latitude, longitude=self.longitude) 493 494 def __eq__(self, other): 495 """Compare two geo points for equality. 496 497 :rtype: bool 498 :returns: True if the points compare equal, else False. 499 """ 500 if not isinstance(other, GeoPoint): 501 return NotImplemented 502 503 return self.latitude == other.latitude and self.longitude == other.longitude 504 505 def __ne__(self, other): 506 """Compare two geo points for inequality. 507 508 :rtype: bool 509 :returns: False if the points compare equal, else True. 510 """ 511 return not self == other 512