1# Copyright 2014-2016 OpenMarket Ltd 2# Copyright 2019 New Vector Ltd 3# Copyright 2020 The Matrix.org Foundation C.I.C. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import abc 18import os 19from typing import ( 20 TYPE_CHECKING, 21 Any, 22 Dict, 23 Generic, 24 Iterable, 25 List, 26 Optional, 27 Sequence, 28 Tuple, 29 Type, 30 TypeVar, 31 Union, 32 overload, 33) 34 35from typing_extensions import Literal 36from unpaddedbase64 import encode_base64 37 38from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions 39from synapse.types import JsonDict, RoomStreamToken 40from synapse.util.caches import intern_dict 41from synapse.util.frozenutils import freeze 42from synapse.util.stringutils import strtobool 43 44if TYPE_CHECKING: 45 from synapse.events.builder import EventBuilder 46 47# Whether we should use frozen_dict in FrozenEvent. Using frozen_dicts prevents 48# bugs where we accidentally share e.g. signature dicts. However, converting a 49# dict to frozen_dicts is expensive. 50# 51# NOTE: This is overridden by the configuration by the Synapse worker apps, but 52# for the sake of tests, it is set here while it cannot be configured on the 53# homeserver object itself. 54 55USE_FROZEN_DICTS = strtobool(os.environ.get("SYNAPSE_USE_FROZEN_DICTS", "0")) 56 57 58T = TypeVar("T") 59 60 61# DictProperty (and DefaultDictProperty) require the classes they're used with to 62# have a _dict property to pull properties from. 63# 64# TODO _DictPropertyInstance should not include EventBuilder but due to 65# https://github.com/python/mypy/issues/5570 it thinks the DictProperty and 66# DefaultDictProperty get applied to EventBuilder when it is in a Union with 67# EventBase. This is the least invasive hack to get mypy to comply. 68# 69# Note that DictProperty/DefaultDictProperty cannot actually be used with 70# EventBuilder as it lacks a _dict property. 71_DictPropertyInstance = Union["_EventInternalMetadata", "EventBase", "EventBuilder"] 72 73 74class DictProperty(Generic[T]): 75 """An object property which delegates to the `_dict` within its parent object.""" 76 77 __slots__ = ["key"] 78 79 def __init__(self, key: str): 80 self.key = key 81 82 @overload 83 def __get__( 84 self, 85 instance: Literal[None], 86 owner: Optional[Type[_DictPropertyInstance]] = None, 87 ) -> "DictProperty": 88 ... 89 90 @overload 91 def __get__( 92 self, 93 instance: _DictPropertyInstance, 94 owner: Optional[Type[_DictPropertyInstance]] = None, 95 ) -> T: 96 ... 97 98 def __get__( 99 self, 100 instance: Optional[_DictPropertyInstance], 101 owner: Optional[Type[_DictPropertyInstance]] = None, 102 ) -> Union[T, "DictProperty"]: 103 # if the property is accessed as a class property rather than an instance 104 # property, return the property itself rather than the value 105 if instance is None: 106 return self 107 try: 108 assert isinstance(instance, (EventBase, _EventInternalMetadata)) 109 return instance._dict[self.key] 110 except KeyError as e1: 111 # We want this to look like a regular attribute error (mostly so that 112 # hasattr() works correctly), so we convert the KeyError into an 113 # AttributeError. 114 # 115 # To exclude the KeyError from the traceback, we explicitly 116 # 'raise from e1.__context__' (which is better than 'raise from None', 117 # because that would omit any *earlier* exceptions). 118 # 119 raise AttributeError( 120 "'%s' has no '%s' property" % (type(instance), self.key) 121 ) from e1.__context__ 122 123 def __set__(self, instance: _DictPropertyInstance, v: T) -> None: 124 assert isinstance(instance, (EventBase, _EventInternalMetadata)) 125 instance._dict[self.key] = v 126 127 def __delete__(self, instance: _DictPropertyInstance) -> None: 128 assert isinstance(instance, (EventBase, _EventInternalMetadata)) 129 try: 130 del instance._dict[self.key] 131 except KeyError as e1: 132 raise AttributeError( 133 "'%s' has no '%s' property" % (type(instance), self.key) 134 ) from e1.__context__ 135 136 137class DefaultDictProperty(DictProperty, Generic[T]): 138 """An extension of DictProperty which provides a default if the property is 139 not present in the parent's _dict. 140 141 Note that this means that hasattr() on the property always returns True. 142 """ 143 144 __slots__ = ["default"] 145 146 def __init__(self, key: str, default: T): 147 super().__init__(key) 148 self.default = default 149 150 @overload 151 def __get__( 152 self, 153 instance: Literal[None], 154 owner: Optional[Type[_DictPropertyInstance]] = None, 155 ) -> "DefaultDictProperty": 156 ... 157 158 @overload 159 def __get__( 160 self, 161 instance: _DictPropertyInstance, 162 owner: Optional[Type[_DictPropertyInstance]] = None, 163 ) -> T: 164 ... 165 166 def __get__( 167 self, 168 instance: Optional[_DictPropertyInstance], 169 owner: Optional[Type[_DictPropertyInstance]] = None, 170 ) -> Union[T, "DefaultDictProperty"]: 171 if instance is None: 172 return self 173 assert isinstance(instance, (EventBase, _EventInternalMetadata)) 174 return instance._dict.get(self.key, self.default) 175 176 177class _EventInternalMetadata: 178 __slots__ = ["_dict", "stream_ordering", "outlier"] 179 180 def __init__(self, internal_metadata_dict: JsonDict): 181 # we have to copy the dict, because it turns out that the same dict is 182 # reused. TODO: fix that 183 self._dict = dict(internal_metadata_dict) 184 185 # the stream ordering of this event. None, until it has been persisted. 186 self.stream_ordering: Optional[int] = None 187 188 # whether this event is an outlier (ie, whether we have the state at that point 189 # in the DAG) 190 self.outlier = False 191 192 out_of_band_membership: DictProperty[bool] = DictProperty("out_of_band_membership") 193 send_on_behalf_of: DictProperty[str] = DictProperty("send_on_behalf_of") 194 recheck_redaction: DictProperty[bool] = DictProperty("recheck_redaction") 195 soft_failed: DictProperty[bool] = DictProperty("soft_failed") 196 proactively_send: DictProperty[bool] = DictProperty("proactively_send") 197 redacted: DictProperty[bool] = DictProperty("redacted") 198 txn_id: DictProperty[str] = DictProperty("txn_id") 199 token_id: DictProperty[int] = DictProperty("token_id") 200 historical: DictProperty[bool] = DictProperty("historical") 201 202 # XXX: These are set by StreamWorkerStore._set_before_and_after. 203 # I'm pretty sure that these are never persisted to the database, so shouldn't 204 # be here 205 before: DictProperty[RoomStreamToken] = DictProperty("before") 206 after: DictProperty[RoomStreamToken] = DictProperty("after") 207 order: DictProperty[Tuple[int, int]] = DictProperty("order") 208 209 def get_dict(self) -> JsonDict: 210 return dict(self._dict) 211 212 def is_outlier(self) -> bool: 213 return self.outlier 214 215 def is_out_of_band_membership(self) -> bool: 216 """Whether this is an out of band membership, like an invite or an invite 217 rejection. This is needed as those events are marked as outliers, but 218 they still need to be processed as if they're new events (e.g. updating 219 invite state in the database, relaying to clients, etc). 220 221 (Added in synapse 0.99.0, so may be unreliable for events received before that) 222 """ 223 return self._dict.get("out_of_band_membership", False) 224 225 def get_send_on_behalf_of(self) -> Optional[str]: 226 """Whether this server should send the event on behalf of another server. 227 This is used by the federation "send_join" API to forward the initial join 228 event for a server in the room. 229 230 returns a str with the name of the server this event is sent on behalf of. 231 """ 232 return self._dict.get("send_on_behalf_of") 233 234 def need_to_check_redaction(self) -> bool: 235 """Whether the redaction event needs to be rechecked when fetching 236 from the database. 237 238 Starting in room v3 redaction events are accepted up front, and later 239 checked to see if the redacter and redactee's domains match. 240 241 If the sender of the redaction event is allowed to redact any event 242 due to auth rules, then this will always return false. 243 """ 244 return self._dict.get("recheck_redaction", False) 245 246 def is_soft_failed(self) -> bool: 247 """Whether the event has been soft failed. 248 249 Soft failed events should be handled as usual, except: 250 1. They should not go down sync or event streams, or generally 251 sent to clients. 252 2. They should not be added to the forward extremities (and 253 therefore not to current state). 254 """ 255 return self._dict.get("soft_failed", False) 256 257 def should_proactively_send(self) -> bool: 258 """Whether the event, if ours, should be sent to other clients and 259 servers. 260 261 This is used for sending dummy events internally. Servers and clients 262 can still explicitly fetch the event. 263 """ 264 return self._dict.get("proactively_send", True) 265 266 def is_redacted(self) -> bool: 267 """Whether the event has been redacted. 268 269 This is used for efficiently checking whether an event has been 270 marked as redacted without needing to make another database call. 271 """ 272 return self._dict.get("redacted", False) 273 274 def is_historical(self) -> bool: 275 """Whether this is a historical message. 276 This is used by the batchsend historical message endpoint and 277 is needed to and mark the event as backfilled and skip some checks 278 like push notifications. 279 """ 280 return self._dict.get("historical", False) 281 282 283class EventBase(metaclass=abc.ABCMeta): 284 @property 285 @abc.abstractmethod 286 def format_version(self) -> int: 287 """The EventFormatVersion implemented by this event""" 288 ... 289 290 def __init__( 291 self, 292 event_dict: JsonDict, 293 room_version: RoomVersion, 294 signatures: Dict[str, Dict[str, str]], 295 unsigned: JsonDict, 296 internal_metadata_dict: JsonDict, 297 rejected_reason: Optional[str], 298 ): 299 assert room_version.event_format == self.format_version 300 301 self.room_version = room_version 302 self.signatures = signatures 303 self.unsigned = unsigned 304 self.rejected_reason = rejected_reason 305 306 self._dict = event_dict 307 308 self.internal_metadata = _EventInternalMetadata(internal_metadata_dict) 309 310 depth: DictProperty[int] = DictProperty("depth") 311 content: DictProperty[JsonDict] = DictProperty("content") 312 hashes: DictProperty[Dict[str, str]] = DictProperty("hashes") 313 origin: DictProperty[str] = DictProperty("origin") 314 origin_server_ts: DictProperty[int] = DictProperty("origin_server_ts") 315 redacts: DefaultDictProperty[Optional[str]] = DefaultDictProperty("redacts", None) 316 room_id: DictProperty[str] = DictProperty("room_id") 317 sender: DictProperty[str] = DictProperty("sender") 318 # TODO state_key should be Optional[str], this is generally asserted in Synapse 319 # by calling is_state() first (which ensures this), but it is hard (not possible?) 320 # to properly annotate that calling is_state() asserts that state_key exists 321 # and is non-None. 322 state_key: DictProperty[str] = DictProperty("state_key") 323 type: DictProperty[str] = DictProperty("type") 324 user_id: DictProperty[str] = DictProperty("sender") 325 326 @property 327 def event_id(self) -> str: 328 raise NotImplementedError() 329 330 @property 331 def membership(self) -> str: 332 return self.content["membership"] 333 334 def is_state(self) -> bool: 335 return hasattr(self, "state_key") and self.state_key is not None 336 337 def get_dict(self) -> JsonDict: 338 d = dict(self._dict) 339 d.update({"signatures": self.signatures, "unsigned": dict(self.unsigned)}) 340 341 return d 342 343 def get(self, key: str, default: Optional[Any] = None) -> Any: 344 return self._dict.get(key, default) 345 346 def get_internal_metadata_dict(self) -> JsonDict: 347 return self.internal_metadata.get_dict() 348 349 def get_pdu_json(self, time_now: Optional[int] = None) -> JsonDict: 350 pdu_json = self.get_dict() 351 352 if time_now is not None and "age_ts" in pdu_json["unsigned"]: 353 age = time_now - pdu_json["unsigned"]["age_ts"] 354 pdu_json.setdefault("unsigned", {})["age"] = int(age) 355 del pdu_json["unsigned"]["age_ts"] 356 357 # This may be a frozen event 358 pdu_json["unsigned"].pop("redacted_because", None) 359 360 return pdu_json 361 362 def get_templated_pdu_json(self) -> JsonDict: 363 """ 364 Return a JSON object suitable for a templated event, as used in the 365 make_{join,leave,knock} workflow. 366 """ 367 # By using _dict directly we don't pull in signatures/unsigned. 368 template_json = dict(self._dict) 369 # The hashes (similar to the signature) need to be recalculated by the 370 # joining/leaving/knocking server after (potentially) modifying the 371 # event. 372 template_json.pop("hashes") 373 374 return template_json 375 376 def __getitem__(self, field: str) -> Optional[Any]: 377 return self._dict[field] 378 379 def __contains__(self, field: str) -> bool: 380 return field in self._dict 381 382 def items(self) -> List[Tuple[str, Optional[Any]]]: 383 return list(self._dict.items()) 384 385 def keys(self) -> Iterable[str]: 386 return self._dict.keys() 387 388 def prev_event_ids(self) -> Sequence[str]: 389 """Returns the list of prev event IDs. The order matches the order 390 specified in the event, though there is no meaning to it. 391 392 Returns: 393 The list of event IDs of this event's prev_events 394 """ 395 return [e for e, _ in self._dict["prev_events"]] 396 397 def auth_event_ids(self) -> Sequence[str]: 398 """Returns the list of auth event IDs. The order matches the order 399 specified in the event, though there is no meaning to it. 400 401 Returns: 402 The list of event IDs of this event's auth_events 403 """ 404 return [e for e, _ in self._dict["auth_events"]] 405 406 def freeze(self) -> None: 407 """'Freeze' the event dict, so it cannot be modified by accident""" 408 409 # this will be a no-op if the event dict is already frozen. 410 self._dict = freeze(self._dict) 411 412 def __str__(self) -> str: 413 return self.__repr__() 414 415 def __repr__(self) -> str: 416 rejection = f"REJECTED={self.rejected_reason}, " if self.rejected_reason else "" 417 418 return ( 419 f"<{self.__class__.__name__} " 420 f"{rejection}" 421 f"event_id={self.event_id}, " 422 f"type={self.get('type')}, " 423 f"state_key={self.get('state_key')}, " 424 f"outlier={self.internal_metadata.is_outlier()}" 425 ">" 426 ) 427 428 429class FrozenEvent(EventBase): 430 format_version = EventFormatVersions.V1 # All events of this type are V1 431 432 def __init__( 433 self, 434 event_dict: JsonDict, 435 room_version: RoomVersion, 436 internal_metadata_dict: Optional[JsonDict] = None, 437 rejected_reason: Optional[str] = None, 438 ): 439 internal_metadata_dict = internal_metadata_dict or {} 440 441 event_dict = dict(event_dict) 442 443 # Signatures is a dict of dicts, and this is faster than doing a 444 # copy.deepcopy 445 signatures = { 446 name: {sig_id: sig for sig_id, sig in sigs.items()} 447 for name, sigs in event_dict.pop("signatures", {}).items() 448 } 449 450 unsigned = dict(event_dict.pop("unsigned", {})) 451 452 # We intern these strings because they turn up a lot (especially when 453 # caching). 454 event_dict = intern_dict(event_dict) 455 456 if USE_FROZEN_DICTS: 457 frozen_dict = freeze(event_dict) 458 else: 459 frozen_dict = event_dict 460 461 self._event_id = event_dict["event_id"] 462 463 super().__init__( 464 frozen_dict, 465 room_version=room_version, 466 signatures=signatures, 467 unsigned=unsigned, 468 internal_metadata_dict=internal_metadata_dict, 469 rejected_reason=rejected_reason, 470 ) 471 472 @property 473 def event_id(self) -> str: 474 return self._event_id 475 476 477class FrozenEventV2(EventBase): 478 format_version = EventFormatVersions.V2 # All events of this type are V2 479 480 def __init__( 481 self, 482 event_dict: JsonDict, 483 room_version: RoomVersion, 484 internal_metadata_dict: Optional[JsonDict] = None, 485 rejected_reason: Optional[str] = None, 486 ): 487 internal_metadata_dict = internal_metadata_dict or {} 488 489 event_dict = dict(event_dict) 490 491 # Signatures is a dict of dicts, and this is faster than doing a 492 # copy.deepcopy 493 signatures = { 494 name: {sig_id: sig for sig_id, sig in sigs.items()} 495 for name, sigs in event_dict.pop("signatures", {}).items() 496 } 497 498 assert "event_id" not in event_dict 499 500 unsigned = dict(event_dict.pop("unsigned", {})) 501 502 # We intern these strings because they turn up a lot (especially when 503 # caching). 504 event_dict = intern_dict(event_dict) 505 506 if USE_FROZEN_DICTS: 507 frozen_dict = freeze(event_dict) 508 else: 509 frozen_dict = event_dict 510 511 self._event_id: Optional[str] = None 512 513 super().__init__( 514 frozen_dict, 515 room_version=room_version, 516 signatures=signatures, 517 unsigned=unsigned, 518 internal_metadata_dict=internal_metadata_dict, 519 rejected_reason=rejected_reason, 520 ) 521 522 @property 523 def event_id(self) -> str: 524 # We have to import this here as otherwise we get an import loop which 525 # is hard to break. 526 from synapse.crypto.event_signing import compute_event_reference_hash 527 528 if self._event_id: 529 return self._event_id 530 self._event_id = "$" + encode_base64(compute_event_reference_hash(self)[1]) 531 return self._event_id 532 533 def prev_event_ids(self) -> Sequence[str]: 534 """Returns the list of prev event IDs. The order matches the order 535 specified in the event, though there is no meaning to it. 536 537 Returns: 538 The list of event IDs of this event's prev_events 539 """ 540 return self._dict["prev_events"] 541 542 def auth_event_ids(self) -> Sequence[str]: 543 """Returns the list of auth event IDs. The order matches the order 544 specified in the event, though there is no meaning to it. 545 546 Returns: 547 The list of event IDs of this event's auth_events 548 """ 549 return self._dict["auth_events"] 550 551 552class FrozenEventV3(FrozenEventV2): 553 """FrozenEventV3, which differs from FrozenEventV2 only in the event_id format""" 554 555 format_version = EventFormatVersions.V3 # All events of this type are V3 556 557 @property 558 def event_id(self) -> str: 559 # We have to import this here as otherwise we get an import loop which 560 # is hard to break. 561 from synapse.crypto.event_signing import compute_event_reference_hash 562 563 if self._event_id: 564 return self._event_id 565 self._event_id = "$" + encode_base64( 566 compute_event_reference_hash(self)[1], urlsafe=True 567 ) 568 return self._event_id 569 570 571def _event_type_from_format_version( 572 format_version: int, 573) -> Type[Union[FrozenEvent, FrozenEventV2, FrozenEventV3]]: 574 """Returns the python type to use to construct an Event object for the 575 given event format version. 576 577 Args: 578 format_version: The event format version 579 580 Returns: 581 type: A type that can be initialized as per the initializer of 582 `FrozenEvent` 583 """ 584 585 if format_version == EventFormatVersions.V1: 586 return FrozenEvent 587 elif format_version == EventFormatVersions.V2: 588 return FrozenEventV2 589 elif format_version == EventFormatVersions.V3: 590 return FrozenEventV3 591 else: 592 raise Exception("No event format %r" % (format_version,)) 593 594 595def make_event_from_dict( 596 event_dict: JsonDict, 597 room_version: RoomVersion = RoomVersions.V1, 598 internal_metadata_dict: Optional[JsonDict] = None, 599 rejected_reason: Optional[str] = None, 600) -> EventBase: 601 """Construct an EventBase from the given event dict""" 602 event_type = _event_type_from_format_version(room_version.event_format) 603 return event_type( 604 event_dict, room_version, internal_metadata_dict or {}, rejected_reason 605 ) 606