1# Copyright 2015, 2016 OpenMarket Ltd 2# Copyright 2017 Vector Creations Ltd 3# Copyright 2018 New Vector Ltd 4# Copyright 2020 The Matrix.org Foundation C.I.C. 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18import itertools 19import logging 20from typing import List, Set 21 22from pkg_resources import ( 23 DistributionNotFound, 24 Requirement, 25 VersionConflict, 26 get_provider, 27) 28 29logger = logging.getLogger(__name__) 30 31 32# REQUIREMENTS is a simple list of requirement specifiers[1], and must be 33# installed. It is passed to setup() as install_requires in setup.py. 34# 35# CONDITIONAL_REQUIREMENTS is the optional dependencies, represented as a dict 36# of lists. The dict key is the optional dependency name and can be passed to 37# pip when installing. The list is a series of requirement specifiers[1] to be 38# installed when that optional dependency requirement is specified. It is passed 39# to setup() as extras_require in setup.py 40# 41# Note that these both represent runtime dependencies (and the versions 42# installed are checked at runtime). 43# 44# Also note that we replicate these constraints in the Synapse Dockerfile while 45# pre-installing dependencies. If these constraints are updated here, the same 46# change should be made in the Dockerfile. 47# 48# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers. 49 50REQUIREMENTS = [ 51 # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0 52 "jsonschema>=3.0.0", 53 # frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41 54 "frozendict>=1", 55 "unpaddedbase64>=1.1.0", 56 "canonicaljson>=1.4.0", 57 # we use the type definitions added in signedjson 1.1. 58 "signedjson>=1.1.0", 59 "pynacl>=1.2.1", 60 "idna>=2.5", 61 # validating SSL certs for IP addresses requires service_identity 18.1. 62 "service_identity>=18.1.0", 63 # Twisted 18.9 introduces some logger improvements that the structured 64 # logger utilises 65 "Twisted>=18.9.0", 66 "treq>=15.1", 67 # Twisted has required pyopenssl 16.0 since about Twisted 16.6. 68 "pyopenssl>=16.0.0", 69 "pyyaml>=3.11", 70 "pyasn1>=0.1.9", 71 "pyasn1-modules>=0.0.7", 72 "bcrypt>=3.1.0", 73 "pillow>=4.3.0", 74 "sortedcontainers>=1.4.4", 75 "pymacaroons>=0.13.0", 76 "msgpack>=0.5.2", 77 "phonenumbers>=8.2.0", 78 # we use GaugeHistogramMetric, which was added in prom-client 0.4.0. 79 "prometheus_client>=0.4.0", 80 # we use `order`, which arrived in attrs 19.2.0. 81 # Note: 21.1.0 broke `/sync`, see #9936 82 "attrs>=19.2.0,!=21.1.0", 83 "netaddr>=0.7.18", 84 "Jinja2>=2.9", 85 "bleach>=1.4.3", 86 "typing-extensions>=3.7.4", 87 # We enforce that we have a `cryptography` version that bundles an `openssl` 88 # with the latest security patches. 89 "cryptography", 90 "ijson>=3.1", 91 "matrix-common==1.0.0", 92] 93 94CONDITIONAL_REQUIREMENTS = { 95 "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], 96 "postgres": [ 97 # we use execute_values with the fetch param, which arrived in psycopg 2.8. 98 "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'", 99 "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'", 100 "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'", 101 ], 102 "saml2": [ 103 "pysaml2>=4.5.0", 104 ], 105 "oidc": ["authlib>=0.14.0"], 106 # systemd-python is necessary for logging to the systemd journal via 107 # `systemd.journal.JournalHandler`, as is documented in 108 # `contrib/systemd/log_config.yaml`. 109 "systemd": ["systemd-python>=231"], 110 "url_preview": ["lxml>=3.5.0"], 111 "sentry": ["sentry-sdk>=0.7.2"], 112 "opentracing": ["jaeger-client>=4.0.0", "opentracing>=2.2.0"], 113 "jwt": ["pyjwt>=1.6.4"], 114 # hiredis is not a *strict* dependency, but it makes things much faster. 115 # (if it is not installed, we fall back to slow code.) 116 "redis": ["txredisapi>=1.4.7", "hiredis"], 117 # Required to use experimental `caches.track_memory_usage` config option. 118 "cache_memory": ["pympler"], 119} 120 121ALL_OPTIONAL_REQUIREMENTS: Set[str] = set() 122 123for name, optional_deps in CONDITIONAL_REQUIREMENTS.items(): 124 # Exclude systemd as it's a system-based requirement. 125 # Exclude lint as it's a dev-based requirement. 126 if name not in ["systemd"]: 127 ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS 128 129 130# ensure there are no double-quote characters in any of the deps (otherwise the 131# 'pip install' incantation in DependencyException will break) 132for dep in itertools.chain( 133 REQUIREMENTS, 134 *CONDITIONAL_REQUIREMENTS.values(), 135): 136 if '"' in dep: 137 raise Exception( 138 "Dependency `%s` contains double-quote; use single-quotes instead" % (dep,) 139 ) 140 141 142def list_requirements(): 143 return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS) 144 145 146class DependencyException(Exception): 147 @property 148 def message(self): 149 return "\n".join( 150 [ 151 "Missing Requirements: %s" % (", ".join(self.dependencies),), 152 "To install run:", 153 " pip install --upgrade --force %s" % (" ".join(self.dependencies),), 154 "", 155 ] 156 ) 157 158 @property 159 def dependencies(self): 160 for i in self.args[0]: 161 yield '"' + i + '"' 162 163 164def check_requirements(for_feature=None): 165 deps_needed = [] 166 errors = [] 167 168 if for_feature: 169 reqs = CONDITIONAL_REQUIREMENTS[for_feature] 170 else: 171 reqs = REQUIREMENTS 172 173 for dependency in reqs: 174 try: 175 _check_requirement(dependency) 176 except VersionConflict as e: 177 deps_needed.append(dependency) 178 errors.append( 179 "Needed %s, got %s==%s" 180 % ( 181 dependency, 182 e.dist.project_name, # type: ignore[attr-defined] # noqa 183 e.dist.version, # type: ignore[attr-defined] # noqa 184 ) 185 ) 186 except DistributionNotFound: 187 deps_needed.append(dependency) 188 if for_feature: 189 errors.append( 190 "Needed %s for the '%s' feature but it was not installed" 191 % (dependency, for_feature) 192 ) 193 else: 194 errors.append("Needed %s but it was not installed" % (dependency,)) 195 196 if not for_feature: 197 # Check the optional dependencies are up to date. We allow them to not be 198 # installed. 199 OPTS: List[str] = sum(CONDITIONAL_REQUIREMENTS.values(), []) 200 201 for dependency in OPTS: 202 try: 203 _check_requirement(dependency) 204 except VersionConflict as e: 205 deps_needed.append(dependency) 206 errors.append( 207 "Needed optional %s, got %s==%s" 208 % ( 209 dependency, 210 e.dist.project_name, # type: ignore[attr-defined] # noqa 211 e.dist.version, # type: ignore[attr-defined] # noqa 212 ) 213 ) 214 except DistributionNotFound: 215 # If it's not found, we don't care 216 pass 217 218 if deps_needed: 219 for err in errors: 220 logging.error(err) 221 222 raise DependencyException(deps_needed) 223 224 225def _check_requirement(dependency_string): 226 """Parses a dependency string, and checks if the specified requirement is installed 227 228 Raises: 229 VersionConflict if the requirement is installed, but with the the wrong version 230 DistributionNotFound if nothing is found to provide the requirement 231 """ 232 req = Requirement.parse(dependency_string) 233 234 # first check if the markers specify that this requirement needs installing 235 if req.marker is not None and not req.marker.evaluate(): 236 # not required for this environment 237 return 238 239 get_provider(req) 240 241 242if __name__ == "__main__": 243 import sys 244 245 sys.stdout.writelines(req + "\n" for req in list_requirements()) 246