1# Copyright 2015, 2016 OpenMarket Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14import platform 15import struct 16import threading 17import typing 18 19from synapse.storage.engines import BaseDatabaseEngine 20from synapse.storage.types import Connection 21 22if typing.TYPE_CHECKING: 23 import sqlite3 # noqa: F401 24 25 26class Sqlite3Engine(BaseDatabaseEngine["sqlite3.Connection"]): 27 def __init__(self, database_module, database_config): 28 super().__init__(database_module, database_config) 29 30 database = database_config.get("args", {}).get("database") 31 self._is_in_memory = database in ( 32 None, 33 ":memory:", 34 ) 35 36 if platform.python_implementation() == "PyPy": 37 # pypy's sqlite3 module doesn't handle bytearrays, convert them 38 # back to bytes. 39 database_module.register_adapter(bytearray, lambda array: bytes(array)) 40 41 # The current max state_group, or None if we haven't looked 42 # in the DB yet. 43 self._current_state_group_id = None 44 self._current_state_group_id_lock = threading.Lock() 45 46 @property 47 def single_threaded(self) -> bool: 48 return True 49 50 @property 51 def can_native_upsert(self): 52 """ 53 Do we support native UPSERTs? This requires SQLite3 3.24+, plus some 54 more work we haven't done yet to tell what was inserted vs updated. 55 """ 56 return self.module.sqlite_version_info >= (3, 24, 0) 57 58 @property 59 def supports_using_any_list(self): 60 """Do we support using `a = ANY(?)` and passing a list""" 61 return False 62 63 @property 64 def supports_returning(self) -> bool: 65 """Do we support the `RETURNING` clause in insert/update/delete?""" 66 return self.module.sqlite_version_info >= (3, 35, 0) 67 68 def check_database(self, db_conn, allow_outdated_version: bool = False): 69 if not allow_outdated_version: 70 version = self.module.sqlite_version_info 71 # Synapse is untested against older SQLite versions, and we don't want 72 # to let users upgrade to a version of Synapse with broken support for their 73 # sqlite version, because it risks leaving them with a half-upgraded db. 74 if version < (3, 22, 0): 75 raise RuntimeError("Synapse requires sqlite 3.22 or above.") 76 77 def check_new_database(self, txn): 78 """Gets called when setting up a brand new database. This allows us to 79 apply stricter checks on new databases versus existing database. 80 """ 81 82 def convert_param_style(self, sql): 83 return sql 84 85 def on_new_connection(self, db_conn): 86 # We need to import here to avoid an import loop. 87 from synapse.storage.prepare_database import prepare_database 88 89 if self._is_in_memory: 90 # In memory databases need to be rebuilt each time. Ideally we'd 91 # reuse the same connection as we do when starting up, but that 92 # would involve using adbapi before we have started the reactor. 93 prepare_database(db_conn, self, config=None) 94 95 db_conn.create_function("rank", 1, _rank) 96 db_conn.execute("PRAGMA foreign_keys = ON;") 97 db_conn.commit() 98 99 def is_deadlock(self, error): 100 return False 101 102 def is_connection_closed(self, conn): 103 return False 104 105 def lock_table(self, txn, table): 106 return 107 108 @property 109 def server_version(self): 110 """Gets a string giving the server version. For example: '3.22.0' 111 112 Returns: 113 string 114 """ 115 return "%i.%i.%i" % self.module.sqlite_version_info 116 117 def in_transaction(self, conn: Connection) -> bool: 118 return conn.in_transaction # type: ignore 119 120 def attempt_to_set_autocommit(self, conn: Connection, autocommit: bool): 121 # Twisted doesn't let us set attributes on the connections, so we can't 122 # set the connection to autocommit mode. 123 pass 124 125 126# Following functions taken from: https://github.com/coleifer/peewee 127 128 129def _parse_match_info(buf): 130 bufsize = len(buf) 131 return [struct.unpack("@I", buf[i : i + 4])[0] for i in range(0, bufsize, 4)] 132 133 134def _rank(raw_match_info): 135 """Handle match_info called w/default args 'pcx' - based on the example rank 136 function http://sqlite.org/fts3.html#appendix_a 137 """ 138 match_info = _parse_match_info(raw_match_info) 139 score = 0.0 140 p, c = match_info[:2] 141 for phrase_num in range(p): 142 phrase_info_idx = 2 + (phrase_num * c * 3) 143 for col_num in range(c): 144 col_idx = phrase_info_idx + (col_num * 3) 145 x1, x2 = match_info[col_idx : col_idx + 2] 146 if x1 > 0: 147 score += float(x1) / x2 148 return score 149