1# bundlecaches.py - utility to deal with pre-computed bundle for servers
2#
3# This software may be used and distributed according to the terms of the
4# GNU General Public License version 2 or any later version.
5
6from .i18n import _
7
8from .thirdparty import attr
9
10from . import (
11    error,
12    requirements as requirementsmod,
13    sslutil,
14    util,
15)
16from .utils import stringutil
17
18urlreq = util.urlreq
19
20CB_MANIFEST_FILE = b'clonebundles.manifest'
21
22
23@attr.s
24class bundlespec(object):
25    compression = attr.ib()
26    wirecompression = attr.ib()
27    version = attr.ib()
28    wireversion = attr.ib()
29    params = attr.ib()
30    contentopts = attr.ib()
31
32
33# Maps bundle version human names to changegroup versions.
34_bundlespeccgversions = {
35    b'v1': b'01',
36    b'v2': b'02',
37    b'packed1': b's1',
38    b'bundle2': b'02',  # legacy
39}
40
41# Maps bundle version with content opts to choose which part to bundle
42_bundlespeccontentopts = {
43    b'v1': {
44        b'changegroup': True,
45        b'cg.version': b'01',
46        b'obsolescence': False,
47        b'phases': False,
48        b'tagsfnodescache': False,
49        b'revbranchcache': False,
50    },
51    b'v2': {
52        b'changegroup': True,
53        b'cg.version': b'02',
54        b'obsolescence': False,
55        b'phases': False,
56        b'tagsfnodescache': True,
57        b'revbranchcache': True,
58    },
59    b'packed1': {b'cg.version': b's1'},
60}
61_bundlespeccontentopts[b'bundle2'] = _bundlespeccontentopts[b'v2']
62
63_bundlespecvariants = {
64    b"streamv2": {
65        b"changegroup": False,
66        b"streamv2": True,
67        b"tagsfnodescache": False,
68        b"revbranchcache": False,
69    }
70}
71
72# Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE.
73_bundlespecv1compengines = {b'gzip', b'bzip2', b'none'}
74
75
76def parsebundlespec(repo, spec, strict=True):
77    """Parse a bundle string specification into parts.
78
79    Bundle specifications denote a well-defined bundle/exchange format.
80    The content of a given specification should not change over time in
81    order to ensure that bundles produced by a newer version of Mercurial are
82    readable from an older version.
83
84    The string currently has the form:
85
86       <compression>-<type>[;<parameter0>[;<parameter1>]]
87
88    Where <compression> is one of the supported compression formats
89    and <type> is (currently) a version string. A ";" can follow the type and
90    all text afterwards is interpreted as URI encoded, ";" delimited key=value
91    pairs.
92
93    If ``strict`` is True (the default) <compression> is required. Otherwise,
94    it is optional.
95
96    Returns a bundlespec object of (compression, version, parameters).
97    Compression will be ``None`` if not in strict mode and a compression isn't
98    defined.
99
100    An ``InvalidBundleSpecification`` is raised when the specification is
101    not syntactically well formed.
102
103    An ``UnsupportedBundleSpecification`` is raised when the compression or
104    bundle type/version is not recognized.
105
106    Note: this function will likely eventually return a more complex data
107    structure, including bundle2 part information.
108    """
109
110    def parseparams(s):
111        if b';' not in s:
112            return s, {}
113
114        params = {}
115        version, paramstr = s.split(b';', 1)
116
117        for p in paramstr.split(b';'):
118            if b'=' not in p:
119                raise error.InvalidBundleSpecification(
120                    _(
121                        b'invalid bundle specification: '
122                        b'missing "=" in parameter: %s'
123                    )
124                    % p
125                )
126
127            key, value = p.split(b'=', 1)
128            key = urlreq.unquote(key)
129            value = urlreq.unquote(value)
130            params[key] = value
131
132        return version, params
133
134    if strict and b'-' not in spec:
135        raise error.InvalidBundleSpecification(
136            _(
137                b'invalid bundle specification; '
138                b'must be prefixed with compression: %s'
139            )
140            % spec
141        )
142
143    if b'-' in spec:
144        compression, version = spec.split(b'-', 1)
145
146        if compression not in util.compengines.supportedbundlenames:
147            raise error.UnsupportedBundleSpecification(
148                _(b'%s compression is not supported') % compression
149            )
150
151        version, params = parseparams(version)
152
153        if version not in _bundlespeccgversions:
154            raise error.UnsupportedBundleSpecification(
155                _(b'%s is not a recognized bundle version') % version
156            )
157    else:
158        # Value could be just the compression or just the version, in which
159        # case some defaults are assumed (but only when not in strict mode).
160        assert not strict
161
162        spec, params = parseparams(spec)
163
164        if spec in util.compengines.supportedbundlenames:
165            compression = spec
166            version = b'v1'
167            # Generaldelta repos require v2.
168            if requirementsmod.GENERALDELTA_REQUIREMENT in repo.requirements:
169                version = b'v2'
170            elif requirementsmod.REVLOGV2_REQUIREMENT in repo.requirements:
171                version = b'v2'
172            # Modern compression engines require v2.
173            if compression not in _bundlespecv1compengines:
174                version = b'v2'
175        elif spec in _bundlespeccgversions:
176            if spec == b'packed1':
177                compression = b'none'
178            else:
179                compression = b'bzip2'
180            version = spec
181        else:
182            raise error.UnsupportedBundleSpecification(
183                _(b'%s is not a recognized bundle specification') % spec
184            )
185
186    # Bundle version 1 only supports a known set of compression engines.
187    if version == b'v1' and compression not in _bundlespecv1compengines:
188        raise error.UnsupportedBundleSpecification(
189            _(b'compression engine %s is not supported on v1 bundles')
190            % compression
191        )
192
193    # The specification for packed1 can optionally declare the data formats
194    # required to apply it. If we see this metadata, compare against what the
195    # repo supports and error if the bundle isn't compatible.
196    if version == b'packed1' and b'requirements' in params:
197        requirements = set(params[b'requirements'].split(b','))
198        missingreqs = requirements - repo.supportedformats
199        if missingreqs:
200            raise error.UnsupportedBundleSpecification(
201                _(b'missing support for repository features: %s')
202                % b', '.join(sorted(missingreqs))
203            )
204
205    # Compute contentopts based on the version
206    contentopts = _bundlespeccontentopts.get(version, {}).copy()
207
208    # Process the variants
209    if b"stream" in params and params[b"stream"] == b"v2":
210        variant = _bundlespecvariants[b"streamv2"]
211        contentopts.update(variant)
212
213    engine = util.compengines.forbundlename(compression)
214    compression, wirecompression = engine.bundletype()
215    wireversion = _bundlespeccgversions[version]
216
217    return bundlespec(
218        compression, wirecompression, version, wireversion, params, contentopts
219    )
220
221
222def parseclonebundlesmanifest(repo, s):
223    """Parses the raw text of a clone bundles manifest.
224
225    Returns a list of dicts. The dicts have a ``URL`` key corresponding
226    to the URL and other keys are the attributes for the entry.
227    """
228    m = []
229    for line in s.splitlines():
230        fields = line.split()
231        if not fields:
232            continue
233        attrs = {b'URL': fields[0]}
234        for rawattr in fields[1:]:
235            key, value = rawattr.split(b'=', 1)
236            key = util.urlreq.unquote(key)
237            value = util.urlreq.unquote(value)
238            attrs[key] = value
239
240            # Parse BUNDLESPEC into components. This makes client-side
241            # preferences easier to specify since you can prefer a single
242            # component of the BUNDLESPEC.
243            if key == b'BUNDLESPEC':
244                try:
245                    bundlespec = parsebundlespec(repo, value)
246                    attrs[b'COMPRESSION'] = bundlespec.compression
247                    attrs[b'VERSION'] = bundlespec.version
248                except error.InvalidBundleSpecification:
249                    pass
250                except error.UnsupportedBundleSpecification:
251                    pass
252
253        m.append(attrs)
254
255    return m
256
257
258def isstreamclonespec(bundlespec):
259    # Stream clone v1
260    if bundlespec.wirecompression == b'UN' and bundlespec.wireversion == b's1':
261        return True
262
263    # Stream clone v2
264    if (
265        bundlespec.wirecompression == b'UN'
266        and bundlespec.wireversion == b'02'
267        and bundlespec.contentopts.get(b'streamv2')
268    ):
269        return True
270
271    return False
272
273
274def filterclonebundleentries(repo, entries, streamclonerequested=False):
275    """Remove incompatible clone bundle manifest entries.
276
277    Accepts a list of entries parsed with ``parseclonebundlesmanifest``
278    and returns a new list consisting of only the entries that this client
279    should be able to apply.
280
281    There is no guarantee we'll be able to apply all returned entries because
282    the metadata we use to filter on may be missing or wrong.
283    """
284    newentries = []
285    for entry in entries:
286        spec = entry.get(b'BUNDLESPEC')
287        if spec:
288            try:
289                bundlespec = parsebundlespec(repo, spec, strict=True)
290
291                # If a stream clone was requested, filter out non-streamclone
292                # entries.
293                if streamclonerequested and not isstreamclonespec(bundlespec):
294                    repo.ui.debug(
295                        b'filtering %s because not a stream clone\n'
296                        % entry[b'URL']
297                    )
298                    continue
299
300            except error.InvalidBundleSpecification as e:
301                repo.ui.debug(stringutil.forcebytestr(e) + b'\n')
302                continue
303            except error.UnsupportedBundleSpecification as e:
304                repo.ui.debug(
305                    b'filtering %s because unsupported bundle '
306                    b'spec: %s\n' % (entry[b'URL'], stringutil.forcebytestr(e))
307                )
308                continue
309        # If we don't have a spec and requested a stream clone, we don't know
310        # what the entry is so don't attempt to apply it.
311        elif streamclonerequested:
312            repo.ui.debug(
313                b'filtering %s because cannot determine if a stream '
314                b'clone bundle\n' % entry[b'URL']
315            )
316            continue
317
318        if b'REQUIRESNI' in entry and not sslutil.hassni:
319            repo.ui.debug(
320                b'filtering %s because SNI not supported\n' % entry[b'URL']
321            )
322            continue
323
324        if b'REQUIREDRAM' in entry:
325            try:
326                requiredram = util.sizetoint(entry[b'REQUIREDRAM'])
327            except error.ParseError:
328                repo.ui.debug(
329                    b'filtering %s due to a bad REQUIREDRAM attribute\n'
330                    % entry[b'URL']
331                )
332                continue
333            actualram = repo.ui.estimatememory()
334            if actualram is not None and actualram * 0.66 < requiredram:
335                repo.ui.debug(
336                    b'filtering %s as it needs more than 2/3 of system memory\n'
337                    % entry[b'URL']
338                )
339                continue
340
341        newentries.append(entry)
342
343    return newentries
344
345
346class clonebundleentry(object):
347    """Represents an item in a clone bundles manifest.
348
349    This rich class is needed to support sorting since sorted() in Python 3
350    doesn't support ``cmp`` and our comparison is complex enough that ``key=``
351    won't work.
352    """
353
354    def __init__(self, value, prefers):
355        self.value = value
356        self.prefers = prefers
357
358    def _cmp(self, other):
359        for prefkey, prefvalue in self.prefers:
360            avalue = self.value.get(prefkey)
361            bvalue = other.value.get(prefkey)
362
363            # Special case for b missing attribute and a matches exactly.
364            if avalue is not None and bvalue is None and avalue == prefvalue:
365                return -1
366
367            # Special case for a missing attribute and b matches exactly.
368            if bvalue is not None and avalue is None and bvalue == prefvalue:
369                return 1
370
371            # We can't compare unless attribute present on both.
372            if avalue is None or bvalue is None:
373                continue
374
375            # Same values should fall back to next attribute.
376            if avalue == bvalue:
377                continue
378
379            # Exact matches come first.
380            if avalue == prefvalue:
381                return -1
382            if bvalue == prefvalue:
383                return 1
384
385            # Fall back to next attribute.
386            continue
387
388        # If we got here we couldn't sort by attributes and prefers. Fall
389        # back to index order.
390        return 0
391
392    def __lt__(self, other):
393        return self._cmp(other) < 0
394
395    def __gt__(self, other):
396        return self._cmp(other) > 0
397
398    def __eq__(self, other):
399        return self._cmp(other) == 0
400
401    def __le__(self, other):
402        return self._cmp(other) <= 0
403
404    def __ge__(self, other):
405        return self._cmp(other) >= 0
406
407    def __ne__(self, other):
408        return self._cmp(other) != 0
409
410
411def sortclonebundleentries(ui, entries):
412    prefers = ui.configlist(b'ui', b'clonebundleprefers')
413    if not prefers:
414        return list(entries)
415
416    def _split(p):
417        if b'=' not in p:
418            hint = _(b"each comma separated item should be key=value pairs")
419            raise error.Abort(
420                _(b"invalid ui.clonebundleprefers item: %s") % p, hint=hint
421            )
422        return p.split(b'=', 1)
423
424    prefers = [_split(p) for p in prefers]
425
426    items = sorted(clonebundleentry(v, prefers) for v in entries)
427    return [i.value for i in items]
428