1# bundlecaches.py - utility to deal with pre-computed bundle for servers 2# 3# This software may be used and distributed according to the terms of the 4# GNU General Public License version 2 or any later version. 5 6from .i18n import _ 7 8from .thirdparty import attr 9 10from . import ( 11 error, 12 requirements as requirementsmod, 13 sslutil, 14 util, 15) 16from .utils import stringutil 17 18urlreq = util.urlreq 19 20CB_MANIFEST_FILE = b'clonebundles.manifest' 21 22 23@attr.s 24class bundlespec(object): 25 compression = attr.ib() 26 wirecompression = attr.ib() 27 version = attr.ib() 28 wireversion = attr.ib() 29 params = attr.ib() 30 contentopts = attr.ib() 31 32 33# Maps bundle version human names to changegroup versions. 34_bundlespeccgversions = { 35 b'v1': b'01', 36 b'v2': b'02', 37 b'packed1': b's1', 38 b'bundle2': b'02', # legacy 39} 40 41# Maps bundle version with content opts to choose which part to bundle 42_bundlespeccontentopts = { 43 b'v1': { 44 b'changegroup': True, 45 b'cg.version': b'01', 46 b'obsolescence': False, 47 b'phases': False, 48 b'tagsfnodescache': False, 49 b'revbranchcache': False, 50 }, 51 b'v2': { 52 b'changegroup': True, 53 b'cg.version': b'02', 54 b'obsolescence': False, 55 b'phases': False, 56 b'tagsfnodescache': True, 57 b'revbranchcache': True, 58 }, 59 b'packed1': {b'cg.version': b's1'}, 60} 61_bundlespeccontentopts[b'bundle2'] = _bundlespeccontentopts[b'v2'] 62 63_bundlespecvariants = { 64 b"streamv2": { 65 b"changegroup": False, 66 b"streamv2": True, 67 b"tagsfnodescache": False, 68 b"revbranchcache": False, 69 } 70} 71 72# Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE. 73_bundlespecv1compengines = {b'gzip', b'bzip2', b'none'} 74 75 76def parsebundlespec(repo, spec, strict=True): 77 """Parse a bundle string specification into parts. 78 79 Bundle specifications denote a well-defined bundle/exchange format. 80 The content of a given specification should not change over time in 81 order to ensure that bundles produced by a newer version of Mercurial are 82 readable from an older version. 83 84 The string currently has the form: 85 86 <compression>-<type>[;<parameter0>[;<parameter1>]] 87 88 Where <compression> is one of the supported compression formats 89 and <type> is (currently) a version string. A ";" can follow the type and 90 all text afterwards is interpreted as URI encoded, ";" delimited key=value 91 pairs. 92 93 If ``strict`` is True (the default) <compression> is required. Otherwise, 94 it is optional. 95 96 Returns a bundlespec object of (compression, version, parameters). 97 Compression will be ``None`` if not in strict mode and a compression isn't 98 defined. 99 100 An ``InvalidBundleSpecification`` is raised when the specification is 101 not syntactically well formed. 102 103 An ``UnsupportedBundleSpecification`` is raised when the compression or 104 bundle type/version is not recognized. 105 106 Note: this function will likely eventually return a more complex data 107 structure, including bundle2 part information. 108 """ 109 110 def parseparams(s): 111 if b';' not in s: 112 return s, {} 113 114 params = {} 115 version, paramstr = s.split(b';', 1) 116 117 for p in paramstr.split(b';'): 118 if b'=' not in p: 119 raise error.InvalidBundleSpecification( 120 _( 121 b'invalid bundle specification: ' 122 b'missing "=" in parameter: %s' 123 ) 124 % p 125 ) 126 127 key, value = p.split(b'=', 1) 128 key = urlreq.unquote(key) 129 value = urlreq.unquote(value) 130 params[key] = value 131 132 return version, params 133 134 if strict and b'-' not in spec: 135 raise error.InvalidBundleSpecification( 136 _( 137 b'invalid bundle specification; ' 138 b'must be prefixed with compression: %s' 139 ) 140 % spec 141 ) 142 143 if b'-' in spec: 144 compression, version = spec.split(b'-', 1) 145 146 if compression not in util.compengines.supportedbundlenames: 147 raise error.UnsupportedBundleSpecification( 148 _(b'%s compression is not supported') % compression 149 ) 150 151 version, params = parseparams(version) 152 153 if version not in _bundlespeccgversions: 154 raise error.UnsupportedBundleSpecification( 155 _(b'%s is not a recognized bundle version') % version 156 ) 157 else: 158 # Value could be just the compression or just the version, in which 159 # case some defaults are assumed (but only when not in strict mode). 160 assert not strict 161 162 spec, params = parseparams(spec) 163 164 if spec in util.compengines.supportedbundlenames: 165 compression = spec 166 version = b'v1' 167 # Generaldelta repos require v2. 168 if requirementsmod.GENERALDELTA_REQUIREMENT in repo.requirements: 169 version = b'v2' 170 elif requirementsmod.REVLOGV2_REQUIREMENT in repo.requirements: 171 version = b'v2' 172 # Modern compression engines require v2. 173 if compression not in _bundlespecv1compengines: 174 version = b'v2' 175 elif spec in _bundlespeccgversions: 176 if spec == b'packed1': 177 compression = b'none' 178 else: 179 compression = b'bzip2' 180 version = spec 181 else: 182 raise error.UnsupportedBundleSpecification( 183 _(b'%s is not a recognized bundle specification') % spec 184 ) 185 186 # Bundle version 1 only supports a known set of compression engines. 187 if version == b'v1' and compression not in _bundlespecv1compengines: 188 raise error.UnsupportedBundleSpecification( 189 _(b'compression engine %s is not supported on v1 bundles') 190 % compression 191 ) 192 193 # The specification for packed1 can optionally declare the data formats 194 # required to apply it. If we see this metadata, compare against what the 195 # repo supports and error if the bundle isn't compatible. 196 if version == b'packed1' and b'requirements' in params: 197 requirements = set(params[b'requirements'].split(b',')) 198 missingreqs = requirements - repo.supportedformats 199 if missingreqs: 200 raise error.UnsupportedBundleSpecification( 201 _(b'missing support for repository features: %s') 202 % b', '.join(sorted(missingreqs)) 203 ) 204 205 # Compute contentopts based on the version 206 contentopts = _bundlespeccontentopts.get(version, {}).copy() 207 208 # Process the variants 209 if b"stream" in params and params[b"stream"] == b"v2": 210 variant = _bundlespecvariants[b"streamv2"] 211 contentopts.update(variant) 212 213 engine = util.compengines.forbundlename(compression) 214 compression, wirecompression = engine.bundletype() 215 wireversion = _bundlespeccgversions[version] 216 217 return bundlespec( 218 compression, wirecompression, version, wireversion, params, contentopts 219 ) 220 221 222def parseclonebundlesmanifest(repo, s): 223 """Parses the raw text of a clone bundles manifest. 224 225 Returns a list of dicts. The dicts have a ``URL`` key corresponding 226 to the URL and other keys are the attributes for the entry. 227 """ 228 m = [] 229 for line in s.splitlines(): 230 fields = line.split() 231 if not fields: 232 continue 233 attrs = {b'URL': fields[0]} 234 for rawattr in fields[1:]: 235 key, value = rawattr.split(b'=', 1) 236 key = util.urlreq.unquote(key) 237 value = util.urlreq.unquote(value) 238 attrs[key] = value 239 240 # Parse BUNDLESPEC into components. This makes client-side 241 # preferences easier to specify since you can prefer a single 242 # component of the BUNDLESPEC. 243 if key == b'BUNDLESPEC': 244 try: 245 bundlespec = parsebundlespec(repo, value) 246 attrs[b'COMPRESSION'] = bundlespec.compression 247 attrs[b'VERSION'] = bundlespec.version 248 except error.InvalidBundleSpecification: 249 pass 250 except error.UnsupportedBundleSpecification: 251 pass 252 253 m.append(attrs) 254 255 return m 256 257 258def isstreamclonespec(bundlespec): 259 # Stream clone v1 260 if bundlespec.wirecompression == b'UN' and bundlespec.wireversion == b's1': 261 return True 262 263 # Stream clone v2 264 if ( 265 bundlespec.wirecompression == b'UN' 266 and bundlespec.wireversion == b'02' 267 and bundlespec.contentopts.get(b'streamv2') 268 ): 269 return True 270 271 return False 272 273 274def filterclonebundleentries(repo, entries, streamclonerequested=False): 275 """Remove incompatible clone bundle manifest entries. 276 277 Accepts a list of entries parsed with ``parseclonebundlesmanifest`` 278 and returns a new list consisting of only the entries that this client 279 should be able to apply. 280 281 There is no guarantee we'll be able to apply all returned entries because 282 the metadata we use to filter on may be missing or wrong. 283 """ 284 newentries = [] 285 for entry in entries: 286 spec = entry.get(b'BUNDLESPEC') 287 if spec: 288 try: 289 bundlespec = parsebundlespec(repo, spec, strict=True) 290 291 # If a stream clone was requested, filter out non-streamclone 292 # entries. 293 if streamclonerequested and not isstreamclonespec(bundlespec): 294 repo.ui.debug( 295 b'filtering %s because not a stream clone\n' 296 % entry[b'URL'] 297 ) 298 continue 299 300 except error.InvalidBundleSpecification as e: 301 repo.ui.debug(stringutil.forcebytestr(e) + b'\n') 302 continue 303 except error.UnsupportedBundleSpecification as e: 304 repo.ui.debug( 305 b'filtering %s because unsupported bundle ' 306 b'spec: %s\n' % (entry[b'URL'], stringutil.forcebytestr(e)) 307 ) 308 continue 309 # If we don't have a spec and requested a stream clone, we don't know 310 # what the entry is so don't attempt to apply it. 311 elif streamclonerequested: 312 repo.ui.debug( 313 b'filtering %s because cannot determine if a stream ' 314 b'clone bundle\n' % entry[b'URL'] 315 ) 316 continue 317 318 if b'REQUIRESNI' in entry and not sslutil.hassni: 319 repo.ui.debug( 320 b'filtering %s because SNI not supported\n' % entry[b'URL'] 321 ) 322 continue 323 324 if b'REQUIREDRAM' in entry: 325 try: 326 requiredram = util.sizetoint(entry[b'REQUIREDRAM']) 327 except error.ParseError: 328 repo.ui.debug( 329 b'filtering %s due to a bad REQUIREDRAM attribute\n' 330 % entry[b'URL'] 331 ) 332 continue 333 actualram = repo.ui.estimatememory() 334 if actualram is not None and actualram * 0.66 < requiredram: 335 repo.ui.debug( 336 b'filtering %s as it needs more than 2/3 of system memory\n' 337 % entry[b'URL'] 338 ) 339 continue 340 341 newentries.append(entry) 342 343 return newentries 344 345 346class clonebundleentry(object): 347 """Represents an item in a clone bundles manifest. 348 349 This rich class is needed to support sorting since sorted() in Python 3 350 doesn't support ``cmp`` and our comparison is complex enough that ``key=`` 351 won't work. 352 """ 353 354 def __init__(self, value, prefers): 355 self.value = value 356 self.prefers = prefers 357 358 def _cmp(self, other): 359 for prefkey, prefvalue in self.prefers: 360 avalue = self.value.get(prefkey) 361 bvalue = other.value.get(prefkey) 362 363 # Special case for b missing attribute and a matches exactly. 364 if avalue is not None and bvalue is None and avalue == prefvalue: 365 return -1 366 367 # Special case for a missing attribute and b matches exactly. 368 if bvalue is not None and avalue is None and bvalue == prefvalue: 369 return 1 370 371 # We can't compare unless attribute present on both. 372 if avalue is None or bvalue is None: 373 continue 374 375 # Same values should fall back to next attribute. 376 if avalue == bvalue: 377 continue 378 379 # Exact matches come first. 380 if avalue == prefvalue: 381 return -1 382 if bvalue == prefvalue: 383 return 1 384 385 # Fall back to next attribute. 386 continue 387 388 # If we got here we couldn't sort by attributes and prefers. Fall 389 # back to index order. 390 return 0 391 392 def __lt__(self, other): 393 return self._cmp(other) < 0 394 395 def __gt__(self, other): 396 return self._cmp(other) > 0 397 398 def __eq__(self, other): 399 return self._cmp(other) == 0 400 401 def __le__(self, other): 402 return self._cmp(other) <= 0 403 404 def __ge__(self, other): 405 return self._cmp(other) >= 0 406 407 def __ne__(self, other): 408 return self._cmp(other) != 0 409 410 411def sortclonebundleentries(ui, entries): 412 prefers = ui.configlist(b'ui', b'clonebundleprefers') 413 if not prefers: 414 return list(entries) 415 416 def _split(p): 417 if b'=' not in p: 418 hint = _(b"each comma separated item should be key=value pairs") 419 raise error.Abort( 420 _(b"invalid ui.clonebundleprefers item: %s") % p, hint=hint 421 ) 422 return p.split(b'=', 1) 423 424 prefers = [_split(p) for p in prefers] 425 426 items = sorted(clonebundleentry(v, prefers) for v in entries) 427 return [i.value for i in items] 428