1# Copyright 2015, 2016 OpenMarket Ltd
2# Copyright 2017 Vector Creations Ltd
3# Copyright 2018 New Vector Ltd
4# Copyright 2020 The Matrix.org Foundation C.I.C.
5#
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10#     http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17
18import itertools
19import logging
20from typing import List, Set
21
22from pkg_resources import (
23    DistributionNotFound,
24    Requirement,
25    VersionConflict,
26    get_provider,
27)
28
29logger = logging.getLogger(__name__)
30
31
32# REQUIREMENTS is a simple list of requirement specifiers[1], and must be
33# installed. It is passed to setup() as install_requires in setup.py.
34#
35# CONDITIONAL_REQUIREMENTS is the optional dependencies, represented as a dict
36# of lists. The dict key is the optional dependency name and can be passed to
37# pip when installing. The list is a series of requirement specifiers[1] to be
38# installed when that optional dependency requirement is specified. It is passed
39# to setup() as extras_require in setup.py
40#
41# Note that these both represent runtime dependencies (and the versions
42# installed are checked at runtime).
43#
44# Also note that we replicate these constraints in the Synapse Dockerfile while
45# pre-installing dependencies. If these constraints are updated here, the same
46# change should be made in the Dockerfile.
47#
48# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers.
49
50REQUIREMENTS = [
51    # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0
52    "jsonschema>=3.0.0",
53    # frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41
54    "frozendict>=1",
55    "unpaddedbase64>=1.1.0",
56    "canonicaljson>=1.4.0",
57    # we use the type definitions added in signedjson 1.1.
58    "signedjson>=1.1.0",
59    "pynacl>=1.2.1",
60    "idna>=2.5",
61    # validating SSL certs for IP addresses requires service_identity 18.1.
62    "service_identity>=18.1.0",
63    # Twisted 18.9 introduces some logger improvements that the structured
64    # logger utilises
65    "Twisted>=18.9.0",
66    "treq>=15.1",
67    # Twisted has required pyopenssl 16.0 since about Twisted 16.6.
68    "pyopenssl>=16.0.0",
69    "pyyaml>=3.11",
70    "pyasn1>=0.1.9",
71    "pyasn1-modules>=0.0.7",
72    "bcrypt>=3.1.0",
73    "pillow>=4.3.0",
74    "sortedcontainers>=1.4.4",
75    "pymacaroons>=0.13.0",
76    "msgpack>=0.5.2",
77    "phonenumbers>=8.2.0",
78    # we use GaugeHistogramMetric, which was added in prom-client 0.4.0.
79    "prometheus_client>=0.4.0",
80    # we use `order`, which arrived in attrs 19.2.0.
81    # Note: 21.1.0 broke `/sync`, see #9936
82    "attrs>=19.2.0,!=21.1.0",
83    "netaddr>=0.7.18",
84    "Jinja2>=2.9",
85    "bleach>=1.4.3",
86    "typing-extensions>=3.7.4",
87    # We enforce that we have a `cryptography` version that bundles an `openssl`
88    # with the latest security patches.
89    "cryptography",
90    "ijson>=3.1",
91    "matrix-common==1.0.0",
92]
93
94CONDITIONAL_REQUIREMENTS = {
95    "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"],
96    "postgres": [
97        # we use execute_values with the fetch param, which arrived in psycopg 2.8.
98        "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'",
99        "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'",
100        "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'",
101    ],
102    "saml2": [
103        "pysaml2>=4.5.0",
104    ],
105    "oidc": ["authlib>=0.14.0"],
106    # systemd-python is necessary for logging to the systemd journal via
107    # `systemd.journal.JournalHandler`, as is documented in
108    # `contrib/systemd/log_config.yaml`.
109    "systemd": ["systemd-python>=231"],
110    "url_preview": ["lxml>=3.5.0"],
111    "sentry": ["sentry-sdk>=0.7.2"],
112    "opentracing": ["jaeger-client>=4.0.0", "opentracing>=2.2.0"],
113    "jwt": ["pyjwt>=1.6.4"],
114    # hiredis is not a *strict* dependency, but it makes things much faster.
115    # (if it is not installed, we fall back to slow code.)
116    "redis": ["txredisapi>=1.4.7", "hiredis"],
117    # Required to use experimental `caches.track_memory_usage` config option.
118    "cache_memory": ["pympler"],
119}
120
121ALL_OPTIONAL_REQUIREMENTS: Set[str] = set()
122
123for name, optional_deps in CONDITIONAL_REQUIREMENTS.items():
124    # Exclude systemd as it's a system-based requirement.
125    # Exclude lint as it's a dev-based requirement.
126    if name not in ["systemd"]:
127        ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS
128
129
130# ensure there are no double-quote characters in any of the deps (otherwise the
131# 'pip install' incantation in DependencyException will break)
132for dep in itertools.chain(
133    REQUIREMENTS,
134    *CONDITIONAL_REQUIREMENTS.values(),
135):
136    if '"' in dep:
137        raise Exception(
138            "Dependency `%s` contains double-quote; use single-quotes instead" % (dep,)
139        )
140
141
142def list_requirements():
143    return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS)
144
145
146class DependencyException(Exception):
147    @property
148    def message(self):
149        return "\n".join(
150            [
151                "Missing Requirements: %s" % (", ".join(self.dependencies),),
152                "To install run:",
153                "    pip install --upgrade --force %s" % (" ".join(self.dependencies),),
154                "",
155            ]
156        )
157
158    @property
159    def dependencies(self):
160        for i in self.args[0]:
161            yield '"' + i + '"'
162
163
164def check_requirements(for_feature=None):
165    deps_needed = []
166    errors = []
167
168    if for_feature:
169        reqs = CONDITIONAL_REQUIREMENTS[for_feature]
170    else:
171        reqs = REQUIREMENTS
172
173    for dependency in reqs:
174        try:
175            _check_requirement(dependency)
176        except VersionConflict as e:
177            deps_needed.append(dependency)
178            errors.append(
179                "Needed %s, got %s==%s"
180                % (
181                    dependency,
182                    e.dist.project_name,  # type: ignore[attr-defined] # noqa
183                    e.dist.version,  # type: ignore[attr-defined] # noqa
184                )
185            )
186        except DistributionNotFound:
187            deps_needed.append(dependency)
188            if for_feature:
189                errors.append(
190                    "Needed %s for the '%s' feature but it was not installed"
191                    % (dependency, for_feature)
192                )
193            else:
194                errors.append("Needed %s but it was not installed" % (dependency,))
195
196    if not for_feature:
197        # Check the optional dependencies are up to date. We allow them to not be
198        # installed.
199        OPTS: List[str] = sum(CONDITIONAL_REQUIREMENTS.values(), [])
200
201        for dependency in OPTS:
202            try:
203                _check_requirement(dependency)
204            except VersionConflict as e:
205                deps_needed.append(dependency)
206                errors.append(
207                    "Needed optional %s, got %s==%s"
208                    % (
209                        dependency,
210                        e.dist.project_name,  # type: ignore[attr-defined] # noqa
211                        e.dist.version,  # type: ignore[attr-defined] # noqa
212                    )
213                )
214            except DistributionNotFound:
215                # If it's not found, we don't care
216                pass
217
218    if deps_needed:
219        for err in errors:
220            logging.error(err)
221
222        raise DependencyException(deps_needed)
223
224
225def _check_requirement(dependency_string):
226    """Parses a dependency string, and checks if the specified requirement is installed
227
228    Raises:
229        VersionConflict if the requirement is installed, but with the the wrong version
230        DistributionNotFound if nothing is found to provide the requirement
231    """
232    req = Requirement.parse(dependency_string)
233
234    # first check if the markers specify that this requirement needs installing
235    if req.marker is not None and not req.marker.evaluate():
236        # not required for this environment
237        return
238
239    get_provider(req)
240
241
242if __name__ == "__main__":
243    import sys
244
245    sys.stdout.writelines(req + "\n" for req in list_requirements())
246