1# -*- coding: utf-8 -*- 2# 3# Copyright (C) 2005-2021 Edgewall Software 4# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de> 5# All rights reserved. 6# 7# This software is licensed as described in the file COPYING, which 8# you should have received as part of this distribution. The terms 9# are also available at https://trac.edgewall.org/wiki/TracLicense. 10# 11# This software consists of voluntary contributions made by many 12# individuals. For the exact contribution history, see the revision 13# history and logs, available at https://trac.edgewall.org/log/. 14# 15# Author: Christopher Lenz <cmlenz@gmx.de> 16 17import os.path 18from abc import ABCMeta, abstractmethod 19from datetime import datetime 20 21from trac.admin import AdminCommandError, IAdminCommandProvider, get_dir_list 22from trac.config import ConfigSection, Option 23from trac.core import * 24from trac.resource import IResourceManager, Resource, ResourceNotFound 25from trac.util import as_bool, native_path 26from trac.util.concurrency import get_thread_id, threading 27from trac.util.datefmt import time_now, utc 28from trac.util.text import exception_to_unicode, printout, to_unicode 29from trac.util.translation import _ 30from trac.web.api import IRequestFilter 31from trac.web.chrome import Chrome, ITemplateProvider, add_warning 32 33 34def is_default(reponame): 35 """Check whether `reponame` is the default repository.""" 36 return not reponame or reponame in ('(default)', _('(default)')) 37 38 39class InvalidRepository(TracError): 40 """Exception raised when a repository is invalid.""" 41 42 43class InvalidConnector(TracError): 44 """Exception raised when a repository connector is invalid.""" 45 46 47class IRepositoryConnector(Interface): 48 """Provide support for a specific version control system.""" 49 50 error = None # place holder for storing relevant error message 51 52 def get_supported_types(): 53 """Return the types of version control systems that are supported. 54 55 Yields `(repotype, priority)` pairs, where `repotype` is used to 56 match against the repository's `type` attribute. 57 58 If multiple provider match a given type, the `priority` is used to 59 choose between them (highest number is highest priority). 60 61 If the `priority` returned is negative, this indicates that the 62 connector for the given `repotype` indeed exists but can't be 63 used for some reason. The `error` property can then be used to 64 store an error message or exception relevant to the problem detected. 65 """ 66 67 def get_repository(repos_type, repos_dir, params): 68 """Return a Repository instance for the given repository type and dir. 69 """ 70 71 72class IRepositoryProvider(Interface): 73 """Provide known named instances of Repository.""" 74 75 def get_repositories(): 76 """Generate repository information for known repositories. 77 78 Repository information is a key,value pair, where the value is 79 a dictionary which must contain at the very least either of 80 the following entries: 81 82 - `'dir'`: the repository directory which can be used by the 83 connector to create a `Repository` instance. This 84 defines a "real" repository. 85 86 - `'alias'`: the name of another repository. This defines an 87 alias to another (real) repository. 88 89 Optional entries: 90 91 - `'type'`: the type of the repository (if not given, the 92 default repository type will be used). 93 94 - `'description'`: a description of the repository (can 95 contain WikiFormatting). 96 97 - `'hidden'`: if set to `'true'`, the repository is hidden 98 from the repository index (default: `'false'`). 99 100 - `'sync_per_request'`: if set to `'true'`, the repository will be 101 synchronized on every request (default: 102 `'false'`). 103 104 - `'url'`: the base URL for checking out the repository. 105 """ 106 107 108class IRepositoryChangeListener(Interface): 109 """Listen for changes in repositories.""" 110 111 def changeset_added(repos, changeset): 112 """Called after a changeset has been added to a repository.""" 113 114 def changeset_modified(repos, changeset, old_changeset): 115 """Called after a changeset has been modified in a repository. 116 117 The `old_changeset` argument contains the metadata of the changeset 118 prior to the modification. It is `None` if the old metadata cannot 119 be retrieved. 120 """ 121 122 123class DbRepositoryProvider(Component): 124 """Component providing repositories registered in the DB.""" 125 126 implements(IRepositoryProvider, IAdminCommandProvider) 127 128 repository_attrs = ('alias', 'description', 'dir', 'hidden', 'name', 129 'sync_per_request', 'type', 'url') 130 131 # IRepositoryProvider methods 132 133 def get_repositories(self): 134 """Retrieve repositories specified in the repository DB table.""" 135 repos = {} 136 for id, name, value in self.env.db_query( 137 "SELECT id, name, value FROM repository WHERE name IN (%s)" 138 % ",".join("'%s'" % each for each in self.repository_attrs)): 139 if value is not None: 140 repos.setdefault(id, {})[name] = value 141 reponames = {} 142 for id, info in repos.items(): 143 if 'name' in info and ('dir' in info or 'alias' in info): 144 info['id'] = id 145 reponames[info['name']] = info 146 info['sync_per_request'] = as_bool(info.get('sync_per_request')) 147 return iter(reponames.items()) 148 149 # IAdminCommandProvider methods 150 151 def get_admin_commands(self): 152 yield ('repository add', '<repos> <dir> [type]', 153 "Add a source repository", 154 self._complete_add, self._do_add) 155 yield ('repository alias', '<name> <target>', 156 "Create an alias for a repository", 157 self._complete_alias, self._do_alias) 158 yield ('repository remove', '<repos>', 159 "Remove a source repository", 160 self._complete_repos, self._do_remove) 161 yield ('repository set', '<repos> <key> <value>', 162 """Set an attribute of a repository 163 164 The following keys are supported: %s 165 """ % ', '.join(self.repository_attrs), 166 self._complete_set, self._do_set) 167 168 def get_reponames(self): 169 rm = RepositoryManager(self.env) 170 return [reponame or '(default)' for reponame 171 in rm.get_all_repositories()] 172 173 def _complete_add(self, args): 174 if len(args) == 2: 175 return get_dir_list(args[-1], True) 176 elif len(args) == 3: 177 return RepositoryManager(self.env).get_supported_types() 178 179 def _complete_alias(self, args): 180 if len(args) == 2: 181 return self.get_reponames() 182 183 def _complete_repos(self, args): 184 if len(args) == 1: 185 return self.get_reponames() 186 187 def _complete_set(self, args): 188 if len(args) == 1: 189 return self.get_reponames() 190 elif len(args) == 2: 191 return self.repository_attrs 192 193 def _do_add(self, reponame, dir, type_=None): 194 self.add_repository(reponame, os.path.abspath(dir), type_) 195 196 def _do_alias(self, reponame, target): 197 self.add_alias(reponame, target) 198 199 def _do_remove(self, reponame): 200 self.remove_repository(reponame) 201 202 def _do_set(self, reponame, key, value): 203 if key not in self.repository_attrs: 204 raise AdminCommandError(_('Invalid key "%(key)s"', key=key)) 205 if key == 'dir': 206 value = os.path.abspath(value) 207 self.modify_repository(reponame, {key: value}) 208 if not reponame: 209 reponame = '(default)' 210 if key == 'dir': 211 printout(_('You should now run "repository resync %(name)s".', 212 name=reponame)) 213 elif key == 'type': 214 printout(_('You may have to run "repository resync %(name)s".', 215 name=reponame)) 216 217 # Public interface 218 219 def add_repository(self, reponame, dir, type_=None): 220 """Add a repository.""" 221 if not os.path.isabs(dir): 222 raise TracError(_("The repository directory must be absolute")) 223 if is_default(reponame): 224 reponame = '' 225 rm = RepositoryManager(self.env) 226 if type_ and type_ not in rm.get_supported_types(): 227 raise TracError(_("The repository type '%(type)s' is not " 228 "supported", type=type_)) 229 with self.env.db_transaction as db: 230 id = rm.get_repository_id(reponame) 231 db.executemany( 232 "INSERT INTO repository (id, name, value) VALUES (%s, %s, %s)", 233 [(id, 'dir', dir), 234 (id, 'type', type_ or '')]) 235 rm.reload_repositories() 236 237 def add_alias(self, reponame, target): 238 """Create an alias repository.""" 239 if is_default(reponame): 240 reponame = '' 241 if is_default(target): 242 target = '' 243 rm = RepositoryManager(self.env) 244 repositories = rm.get_all_repositories() 245 if target not in repositories: 246 raise TracError(_("Repository \"%(repo)s\" doesn't exist", 247 repo=target or '(default)')) 248 if 'alias' in repositories[target]: 249 raise TracError(_('Cannot create an alias to the alias "%(repo)s"', 250 repo=target or '(default)')) 251 with self.env.db_transaction as db: 252 id = rm.get_repository_id(reponame) 253 db.executemany( 254 "INSERT INTO repository (id, name, value) VALUES (%s, %s, %s)", 255 [(id, 'dir', None), 256 (id, 'alias', target)]) 257 rm.reload_repositories() 258 259 def remove_repository(self, reponame): 260 """Remove a repository.""" 261 if is_default(reponame): 262 reponame = '' 263 rm = RepositoryManager(self.env) 264 repositories = rm.get_all_repositories() 265 if any(reponame == repos.get('alias') 266 for repos in repositories.values()): 267 raise TracError(_('Cannot remove the repository "%(repos)s" used ' 268 'in aliases', repos=reponame or '(default)')) 269 with self.env.db_transaction as db: 270 id = rm.get_repository_id(reponame) 271 db("DELETE FROM repository WHERE id=%s", (id,)) 272 db("DELETE FROM revision WHERE repos=%s", (id,)) 273 db("DELETE FROM node_change WHERE repos=%s", (id,)) 274 rm.reload_repositories() 275 276 def modify_repository(self, reponame, changes): 277 """Modify attributes of a repository.""" 278 if is_default(reponame): 279 reponame = '' 280 new_reponame = changes.get('name', reponame) 281 if is_default(new_reponame): 282 new_reponame = '' 283 rm = RepositoryManager(self.env) 284 if reponame != new_reponame: 285 repositories = rm.get_all_repositories() 286 if any(reponame == repos.get('alias') 287 for repos in repositories.values()): 288 raise TracError(_('Cannot rename the repository "%(repos)s" ' 289 'used in aliases', 290 repos=reponame or '(default)')) 291 with self.env.db_transaction as db: 292 id = rm.get_repository_id(reponame) 293 if reponame != new_reponame: 294 if db("""SELECT id FROM repository WHERE name='name' AND 295 value=%s""", (new_reponame,)): 296 raise TracError(_('The repository "%(name)s" already ' 297 'exists.', 298 name=new_reponame or '(default)')) 299 for (k, v) in changes.items(): 300 if k not in self.repository_attrs: 301 continue 302 if k in ('alias', 'name') and is_default(v): 303 v = '' 304 if k in ('hidden', 'sync_per_request'): 305 v = '1' if as_bool(v) else None 306 if k == 'dir' and not os.path.isabs(native_path(v)): 307 raise TracError(_("The repository directory must be " 308 "absolute")) 309 db("UPDATE repository SET value=%s WHERE id=%s AND name=%s", 310 (v, id, k)) 311 if not db( 312 "SELECT value FROM repository WHERE id=%s AND name=%s", 313 (id, k)): 314 db("""INSERT INTO repository (id, name, value) 315 VALUES (%s, %s, %s) 316 """, (id, k, v)) 317 rm.reload_repositories() 318 319 320class RepositoryManager(Component): 321 """Version control system manager.""" 322 323 implements(IRequestFilter, IResourceManager, IRepositoryProvider, 324 ITemplateProvider) 325 326 changeset_realm = 'changeset' 327 source_realm = 'source' 328 repository_realm = 'repository' 329 330 connectors = ExtensionPoint(IRepositoryConnector) 331 providers = ExtensionPoint(IRepositoryProvider) 332 change_listeners = ExtensionPoint(IRepositoryChangeListener) 333 334 repositories_section = ConfigSection('repositories', 335 """One of the methods for registering repositories is to 336 populate the `[repositories]` section of `trac.ini`. 337 338 This is especially suited for setting up aliases, using a 339 [TracIni#GlobalConfiguration shared configuration], or specifying 340 repositories at the time of environment creation. 341 342 See [TracRepositoryAdmin#ReposTracIni TracRepositoryAdmin] for 343 details on the format of this section, and look elsewhere on the 344 page for information on other repository providers. 345 """) 346 347 default_repository_type = Option('versioncontrol', 348 'default_repository_type', 'svn', 349 """Default repository connector type. 350 351 This is used as the default repository type for repositories 352 defined in the [TracIni#repositories-section repositories] section 353 or using the "Repositories" admin panel. 354 """) 355 356 def __init__(self): 357 self._cache = {} 358 self._lock = threading.Lock() 359 self._connectors = None 360 self._all_repositories = None 361 362 # IRequestFilter methods 363 364 def pre_process_request(self, req, handler): 365 if handler is not Chrome(self.env): 366 for repo_info in self.get_all_repositories().values(): 367 if not as_bool(repo_info.get('sync_per_request')): 368 continue 369 start = time_now() 370 repo_name = repo_info['name'] or '(default)' 371 try: 372 repo = self.get_repository(repo_info['name']) 373 repo.sync() 374 except InvalidConnector: 375 continue 376 except TracError as e: 377 add_warning(req, 378 _("Can't synchronize with repository \"%(name)s\" " 379 "(%(error)s). Look in the Trac log for more " 380 "information.", name=repo_name, 381 error=to_unicode(e))) 382 except Exception as e: 383 add_warning(req, 384 _("Failed to sync with repository \"%(name)s\": " 385 "%(error)s; repository information may be out of " 386 "date. Look in the Trac log for more information " 387 "including mitigation strategies.", 388 name=repo_name, error=to_unicode(e))) 389 self.log.error( 390 "Failed to sync with repository \"%s\"; You may be " 391 "able to reduce the impact of this issue by " 392 "configuring the sync_per_request option; see " 393 "https://trac.edgewall.org/wiki/TracRepositoryAdmin" 394 "#ExplicitSync for more detail: %s", repo_name, 395 exception_to_unicode(e, traceback=True)) 396 self.log.info("Synchronized '%s' repository in %0.2f seconds", 397 repo_name, time_now() - start) 398 return handler 399 400 def post_process_request(self, req, template, data, metadata): 401 return template, data, metadata 402 403 # IResourceManager methods 404 405 def get_resource_realms(self): 406 yield self.changeset_realm 407 yield self.source_realm 408 yield self.repository_realm 409 410 def get_resource_description(self, resource, format=None, **kwargs): 411 if resource.realm == self.changeset_realm: 412 parent = resource.parent 413 reponame = parent and parent.id 414 id = resource.id 415 if reponame: 416 return _("Changeset %(rev)s in %(repo)s", rev=id, repo=reponame) 417 else: 418 return _("Changeset %(rev)s", rev=id) 419 elif resource.realm == self.source_realm: 420 parent = resource.parent 421 reponame = parent and parent.id 422 id = resource.id 423 version = '' 424 if format == 'summary': 425 repos = self.get_repository(reponame) 426 node = repos.get_node(resource.id, resource.version) 427 if node.isdir: 428 kind = _("directory") 429 elif node.isfile: 430 kind = _("file") 431 if resource.version: 432 version = _(" at version %(rev)s", rev=resource.version) 433 else: 434 kind = _("path") 435 if resource.version: 436 version = '@%s' % resource.version 437 in_repo = _(" in %(repo)s", repo=reponame) if reponame else '' 438 # TRANSLATOR: file /path/to/file.py at version 13 in reponame 439 return _('%(kind)s %(id)s%(at_version)s%(in_repo)s', 440 kind=kind, id=id, at_version=version, in_repo=in_repo) 441 elif resource.realm == self.repository_realm: 442 if not resource.id: 443 return _("Default repository") 444 return _("Repository %(repo)s", repo=resource.id) 445 446 def get_resource_url(self, resource, href, **kwargs): 447 if resource.realm == self.changeset_realm: 448 parent = resource.parent 449 return href.changeset(resource.id, parent and parent.id or None) 450 elif resource.realm == self.source_realm: 451 parent = resource.parent 452 return href.browser(parent and parent.id or None, resource.id, 453 rev=resource.version or None) 454 elif resource.realm == self.repository_realm: 455 return href.browser(resource.id or None) 456 457 def resource_exists(self, resource): 458 if resource.realm == self.repository_realm: 459 reponame = resource.id 460 else: 461 reponame = resource.parent.id 462 repos = RepositoryManager(self.env).get_repository(reponame) 463 if not repos: 464 return False 465 if resource.realm == self.changeset_realm: 466 try: 467 repos.get_changeset(resource.id) 468 return True 469 except NoSuchChangeset: 470 return False 471 elif resource.realm == self.source_realm: 472 try: 473 repos.get_node(resource.id, resource.version) 474 return True 475 except NoSuchNode: 476 return False 477 elif resource.realm == self.repository_realm: 478 return True 479 480 # IRepositoryProvider methods 481 482 def get_repositories(self): 483 """Retrieve repositories specified in TracIni. 484 485 The `[repositories]` section can be used to specify a list 486 of repositories. 487 """ 488 repositories = self.repositories_section 489 reponames = {} 490 # first pass to gather the <name>.dir entries 491 for option in repositories: 492 if option.endswith('.dir') and repositories.get(option): 493 reponames[option[:-4]] = {'sync_per_request': False} 494 # second pass to gather aliases 495 for option in repositories: 496 alias = repositories.get(option) 497 if '.' not in option: # Support <alias> = <repo> syntax 498 option += '.alias' 499 if option.endswith('.alias') and alias in reponames: 500 reponames.setdefault(option[:-6], {})['alias'] = alias 501 # third pass to gather the <name>.<detail> entries 502 for option in repositories: 503 if '.' in option: 504 name, detail = option.rsplit('.', 1) 505 if name in reponames and detail != 'alias': 506 reponames[name][detail] = repositories.get(option) 507 508 for reponame, info in reponames.items(): 509 yield (reponame, info) 510 511 # ITemplateProvider methods 512 513 def get_htdocs_dirs(self): 514 return [] 515 516 def get_templates_dirs(self): 517 from pkg_resources import resource_filename 518 return [resource_filename('trac.versioncontrol', 'templates')] 519 520 # Public API methods 521 522 def get_supported_types(self): 523 """Return the list of supported repository types.""" 524 types = {type_ 525 for connector in self.connectors 526 for (type_, prio) in connector.get_supported_types() or [] 527 if prio >= 0} 528 return list(types) 529 530 def get_repositories_by_dir(self, directory): 531 """Retrieve the repositories based on the given directory. 532 533 :param directory: the key for identifying the repositories. 534 :return: list of `Repository` instances. 535 """ 536 directory = os.path.join(os.path.normcase(native_path(directory)), '') 537 repositories = [] 538 for reponame, repoinfo in self.get_all_repositories().items(): 539 dir = native_path(repoinfo.get('dir')) 540 if dir: 541 dir = os.path.join(os.path.normcase(dir), '') 542 if dir.startswith(directory): 543 repos = self.get_repository(reponame) 544 if repos: 545 repositories.append(repos) 546 return repositories 547 548 def get_repository_id(self, reponame): 549 """Return a unique id for the given repository name. 550 551 This will create and save a new id if none is found. 552 553 Note: this should probably be renamed as we're dealing 554 exclusively with *db* repository ids here. 555 """ 556 with self.env.db_transaction as db: 557 for id, in db( 558 "SELECT id FROM repository WHERE name='name' AND value=%s", 559 (reponame,)): 560 return id 561 562 id = db("SELECT COALESCE(MAX(id), 0) FROM repository")[0][0] + 1 563 db("INSERT INTO repository (id, name, value) VALUES (%s, %s, %s)", 564 (id, 'name', reponame)) 565 return id 566 567 def get_repository(self, reponame): 568 """Retrieve the appropriate `Repository` for the given 569 repository name. 570 571 :param reponame: the key for specifying the repository. 572 If no name is given, take the default 573 repository. 574 :return: if no corresponding repository was defined, 575 simply return `None`. 576 577 :raises InvalidConnector: if the repository connector cannot be 578 opened. 579 :raises InvalidRepository: if the repository cannot be opened. 580 """ 581 reponame = reponame or '' 582 repoinfo = self.get_all_repositories().get(reponame, {}) 583 if 'alias' in repoinfo: 584 reponame = repoinfo['alias'] 585 repoinfo = self.get_all_repositories().get(reponame, {}) 586 rdir = native_path(repoinfo.get('dir')) 587 if not rdir: 588 return None 589 rtype = repoinfo.get('type') or self.default_repository_type 590 591 # get a Repository for the reponame (use a thread-level cache) 592 with self.env.db_transaction: # prevent possible deadlock, see #4465 593 with self._lock: 594 tid = get_thread_id() 595 if tid in self._cache: 596 repositories = self._cache[tid] 597 else: 598 repositories = self._cache[tid] = {} 599 repos = repositories.get(reponame) 600 if not repos: 601 if not os.path.isabs(rdir): 602 rdir = os.path.join(self.env.path, rdir) 603 connector = self._get_connector(rtype) 604 repos = connector.get_repository(rtype, rdir, 605 repoinfo.copy()) 606 repositories[reponame] = repos 607 return repos 608 609 def get_repository_by_path(self, path): 610 """Retrieve a matching `Repository` for the given `path`. 611 612 :param path: the eventually scoped repository-scoped path 613 :return: a `(reponame, repos, path)` triple, where `path` is 614 the remaining part of `path` once the `reponame` has 615 been truncated, if needed. 616 """ 617 matches = [] 618 path = path.strip('/') + '/' if path else '/' 619 for reponame in self.get_all_repositories(): 620 stripped_reponame = reponame.strip('/') + '/' 621 if path.startswith(stripped_reponame): 622 matches.append((len(stripped_reponame), reponame)) 623 if matches: 624 matches.sort() 625 length, reponame = matches[-1] 626 path = path[length:] 627 else: 628 reponame = '' 629 return (reponame, self.get_repository(reponame), 630 path.rstrip('/') or '/') 631 632 def get_default_repository(self, context): 633 """Recover the appropriate repository from the current context. 634 635 Lookup the closest source or changeset resource in the context 636 hierarchy and return the name of its associated repository. 637 """ 638 while context: 639 if context.resource.realm in (self.source_realm, 640 self.changeset_realm) and \ 641 context.resource.parent: 642 return context.resource.parent.id 643 context = context.parent 644 645 def get_all_repositories(self): 646 """Return a dictionary of repository information, indexed by name.""" 647 if not self._all_repositories: 648 all_repositories = {} 649 for provider in self.providers: 650 for reponame, info in provider.get_repositories() or []: 651 if reponame in all_repositories: 652 self.log.warning("Discarding duplicate repository " 653 "'%s'", reponame) 654 else: 655 info['name'] = reponame 656 if 'id' not in info: 657 info['id'] = self.get_repository_id(reponame) 658 all_repositories[reponame] = info 659 self._all_repositories = all_repositories 660 return self._all_repositories 661 662 def get_real_repositories(self): 663 """Return a sorted list of all real repositories (i.e. excluding 664 aliases). 665 """ 666 repositories = set() 667 for reponame in self.get_all_repositories(): 668 try: 669 repos = self.get_repository(reponame) 670 except TracError: 671 pass # Skip invalid repositories 672 else: 673 if repos is not None: 674 repositories.add(repos) 675 return sorted(repositories, key=lambda r: r.reponame) 676 677 def reload_repositories(self): 678 """Reload the repositories from the providers.""" 679 with self._lock: 680 # FIXME: trac-admin doesn't reload the environment 681 self._cache = {} 682 self._all_repositories = None 683 self.config.touch() # Force environment reload 684 685 def notify(self, event, reponame, revs): 686 """Notify repositories and change listeners about repository events. 687 688 The supported events are the names of the methods defined in the 689 `IRepositoryChangeListener` interface. 690 """ 691 self.log.debug("Event %s on repository '%s' for changesets %r", 692 event, reponame or '(default)', revs) 693 694 # Notify a repository by name, and all repositories with the same 695 # base, or all repositories by base or by repository dir 696 repos = self.get_repository(reponame) 697 repositories = [] 698 if repos: 699 base = repos.get_base() 700 else: 701 dir = os.path.abspath(reponame) 702 repositories = self.get_repositories_by_dir(dir) 703 if repositories: 704 base = None 705 else: 706 base = reponame 707 if base: 708 repositories = [r for r in self.get_real_repositories() 709 if r.get_base() == base] 710 if not repositories: 711 self.log.warning("Found no repositories matching '%s' base.", 712 base or reponame) 713 return [_("Repository '%(repo)s' not found", 714 repo=reponame or _("(default)"))] 715 716 errors = [] 717 for repos in sorted(repositories, key=lambda r: r.reponame): 718 reponame = repos.reponame or '(default)' 719 repos.sync() 720 for rev in revs: 721 args = [] 722 if event == 'changeset_modified': 723 try: 724 old_changeset = repos.sync_changeset(rev) 725 except NoSuchChangeset as e: 726 errors.append(exception_to_unicode(e)) 727 self.log.warning( 728 "No changeset '%s' found in repository '%s'. " 729 "Skipping subscribers for event %s", 730 rev, reponame, event) 731 continue 732 else: 733 args.append(old_changeset) 734 try: 735 changeset = repos.get_changeset(rev) 736 except NoSuchChangeset: 737 try: 738 repos.sync_changeset(rev) 739 changeset = repos.get_changeset(rev) 740 except NoSuchChangeset as e: 741 errors.append(exception_to_unicode(e)) 742 self.log.warning( 743 "No changeset '%s' found in repository '%s'. " 744 "Skipping subscribers for event %s", 745 rev, reponame, event) 746 continue 747 self.log.debug("Event %s on repository '%s' for revision '%s'", 748 event, reponame, rev) 749 for listener in self.change_listeners: 750 getattr(listener, event)(repos, changeset, *args) 751 return errors 752 753 def shutdown(self, tid=None): 754 """Free `Repository` instances bound to a given thread identifier""" 755 if tid: 756 assert tid == get_thread_id() 757 with self._lock: 758 repositories = self._cache.pop(tid, {}) 759 for reponame, repos in repositories.items(): 760 repos.close() 761 762 def read_file_by_path(self, path): 763 """Read the file specified by `path` 764 765 :param path: the repository-scoped path. The repository revision may 766 specified by appending `@` followed by the revision, 767 otherwise the HEAD revision is assumed. 768 :return: the file content as a `str` string. `None` is returned if 769 the file is not found. 770 771 :since: 1.2.2 772 """ 773 repos, path = self.get_repository_by_path(path)[1:] 774 if not repos: 775 return None 776 rev = None 777 if '@' in path: 778 path, rev = path.split('@', 1) 779 try: 780 node = repos.get_node(path, rev) 781 except (NoSuchChangeset, NoSuchNode): 782 return None 783 content = node.get_content() 784 if content: 785 return to_unicode(content.read()) 786 787 # private methods 788 789 def _get_connector(self, rtype): 790 """Retrieve the appropriate connector for the given repository type. 791 792 Note that the self._lock must be held when calling this method. 793 """ 794 if self._connectors is None: 795 # build an environment-level cache for the preferred connectors 796 self._connectors = {} 797 for connector in self.connectors: 798 for type_, prio in connector.get_supported_types() or []: 799 keep = (connector, prio) 800 if type_ in self._connectors and \ 801 prio <= self._connectors[type_][1]: 802 keep = None 803 if keep: 804 self._connectors[type_] = keep 805 if rtype in self._connectors: 806 connector, prio = self._connectors[rtype] 807 if prio >= 0: # no error condition 808 return connector 809 else: 810 raise InvalidConnector( 811 _('Unsupported version control system "%(name)s"' 812 ': %(error)s', name=rtype, 813 error=to_unicode(connector.error))) 814 else: 815 raise InvalidConnector( 816 _('Unsupported version control system "%(name)s": ' 817 'Can\'t find an appropriate component, maybe the ' 818 'corresponding plugin was not enabled? ', name=rtype)) 819 820 821class NoSuchChangeset(ResourceNotFound): 822 def __init__(self, rev): 823 ResourceNotFound.__init__(self, 824 _('No changeset %(rev)s in the repository', 825 rev=rev), 826 _('No such changeset')) 827 828 829class NoSuchNode(ResourceNotFound): 830 def __init__(self, path, rev, msg=None): 831 if msg is None: 832 msg = _("No node %(path)s at revision %(rev)s", path=path, rev=rev) 833 else: 834 msg = _("%(msg)s: No node %(path)s at revision %(rev)s", 835 msg=msg, path=path, rev=rev) 836 ResourceNotFound.__init__(self, msg, _('No such node')) 837 838 839class Repository(object, metaclass=ABCMeta): 840 """Base class for a repository provided by a version control system.""" 841 842 has_linear_changesets = False 843 844 scope = '/' 845 846 realm = RepositoryManager.repository_realm 847 848 @property 849 def resource(self): 850 return Resource(self.realm, self.reponame) 851 852 def __init__(self, name, params, log): 853 """Initialize a repository. 854 855 :param name: a unique name identifying the repository, usually a 856 type-specific prefix followed by the path to the 857 repository. 858 :param params: a `dict` of parameters for the repository. Contains 859 the name of the repository under the key "name" and 860 the surrogate key that identifies the repository in 861 the database under the key "id". 862 :param log: a logger instance. 863 864 :raises InvalidRepository: if the repository cannot be opened. 865 """ 866 self.name = name 867 self.params = params 868 self.reponame = params['name'] 869 self.id = params['id'] 870 self.log = log 871 872 def __repr__(self): 873 return '<%s %r %r %r>' % (self.__class__.__name__, 874 self.id, self.name, self.scope) 875 876 @abstractmethod 877 def close(self): 878 """Close the connection to the repository.""" 879 pass 880 881 def get_base(self): 882 """Return the name of the base repository for this repository. 883 884 This function returns the name of the base repository to which scoped 885 repositories belong. For non-scoped repositories, it returns the 886 repository name. 887 """ 888 return self.name 889 890 def clear(self, youngest_rev=None): 891 """Clear any data that may have been cached in instance properties. 892 893 `youngest_rev` can be specified as a way to force the value 894 of the `youngest_rev` property (''will change in 0.12''). 895 """ 896 pass 897 898 def sync(self, rev_callback=None, clean=False): 899 """Perform a sync of the repository cache, if relevant. 900 901 If given, `rev_callback` must be a callable taking a `rev` parameter. 902 The backend will call this function for each `rev` it decided to 903 synchronize, once the synchronization changes are committed to the 904 cache. When `clean` is `True`, the cache is cleaned first. 905 """ 906 pass 907 908 def sync_changeset(self, rev): 909 """Resync the repository cache for the given `rev`, if relevant. 910 911 Returns a "metadata-only" changeset containing the metadata prior to 912 the resync, or `None` if the old values cannot be retrieved (typically 913 when the repository is not cached). 914 """ 915 return None 916 917 def get_quickjump_entries(self, rev): 918 """Generate a list of interesting places in the repository. 919 920 `rev` might be used to restrict the list of available locations, 921 but in general it's best to produce all known locations. 922 923 The generated results must be of the form (category, name, path, rev). 924 """ 925 return [] 926 927 def get_path_url(self, path, rev): 928 """Return the repository URL for the given path and revision. 929 930 The returned URL can be `None`, meaning that no URL has been specified 931 for the repository, an absolute URL, or a scheme-relative URL starting 932 with `//`, in which case the scheme of the request should be prepended. 933 """ 934 return None 935 936 @abstractmethod 937 def get_changeset(self, rev): 938 """Retrieve a Changeset corresponding to the given revision `rev`.""" 939 pass 940 941 def get_changeset_uid(self, rev): 942 """Return a globally unique identifier for the ''rev'' changeset. 943 944 Two changesets from different repositories can sometimes refer to 945 the ''very same'' changeset (e.g. the repositories are clones). 946 """ 947 948 def get_changesets(self, start, stop): 949 """Generate Changeset belonging to the given time period (start, stop). 950 """ 951 rev = self.youngest_rev 952 while rev: 953 chgset = self.get_changeset(rev) 954 if chgset.date < start: 955 return 956 if chgset.date < stop: 957 yield chgset 958 rev = self.previous_rev(rev) 959 960 def has_node(self, path, rev=None): 961 """Tell if there's a node at the specified (path,rev) combination. 962 963 When `rev` is `None`, the latest revision is implied. 964 """ 965 try: 966 self.get_node(path, rev) 967 return True 968 except TracError: 969 return False 970 971 @abstractmethod 972 def get_node(self, path, rev=None): 973 """Retrieve a Node from the repository at the given path. 974 975 A Node represents a directory or a file at a given revision in the 976 repository. 977 If the `rev` parameter is specified, the Node corresponding to that 978 revision is returned, otherwise the Node corresponding to the youngest 979 revision is returned. 980 """ 981 pass 982 983 @abstractmethod 984 def get_oldest_rev(self): 985 """Return the oldest revision stored in the repository.""" 986 pass 987 oldest_rev = property(lambda self: self.get_oldest_rev()) 988 989 @abstractmethod 990 def get_youngest_rev(self): 991 """Return the youngest revision in the repository.""" 992 pass 993 youngest_rev = property(lambda self: self.get_youngest_rev()) 994 995 @abstractmethod 996 def previous_rev(self, rev, path=''): 997 """Return the revision immediately preceding the specified revision. 998 999 If `path` is given, filter out ancestor revisions having no changes 1000 below `path`. 1001 1002 In presence of multiple parents, this follows the first parent. 1003 """ 1004 pass 1005 1006 @abstractmethod 1007 def next_rev(self, rev, path=''): 1008 """Return the revision immediately following the specified revision. 1009 1010 If `path` is given, filter out descendant revisions having no changes 1011 below `path`. 1012 1013 In presence of multiple children, this follows the first child. 1014 """ 1015 pass 1016 1017 def parent_revs(self, rev): 1018 """Return a list of parents of the specified revision.""" 1019 parent = self.previous_rev(rev) 1020 return [parent] if parent is not None else [] 1021 1022 @abstractmethod 1023 def rev_older_than(self, rev1, rev2): 1024 """Provides a total order over revisions. 1025 1026 Return `True` if `rev1` is an ancestor of `rev2`. 1027 """ 1028 pass 1029 1030 @abstractmethod 1031 def get_path_history(self, path, rev=None, limit=None): 1032 """Retrieve all the revisions containing this path. 1033 1034 If given, `rev` is used as a starting point (i.e. no revision 1035 ''newer'' than `rev` should be returned). 1036 The result format should be the same as the one of Node.get_history() 1037 """ 1038 pass 1039 1040 @abstractmethod 1041 def normalize_path(self, path): 1042 """Return a canonical representation of path in the repos.""" 1043 pass 1044 1045 @abstractmethod 1046 def normalize_rev(self, rev): 1047 """Return a (unique) canonical representation of a revision. 1048 1049 It's up to the backend to decide which string values of `rev` 1050 (usually provided by the user) should be accepted, and how they 1051 should be normalized. Some backends may for instance want to match 1052 against known tags or branch names. 1053 1054 In addition, if `rev` is `None` or '', the youngest revision should 1055 be returned. 1056 1057 :raise NoSuchChangeset: If the given `rev` isn't found. 1058 """ 1059 pass 1060 1061 def short_rev(self, rev): 1062 """Return a compact string representation of a revision in the 1063 repos. 1064 1065 :raise NoSuchChangeset: If the given `rev` isn't found. 1066 :since 1.2: Always returns a string or `None`. 1067 """ 1068 norm_rev = self.normalize_rev(rev) 1069 return str(norm_rev) if norm_rev is not None else norm_rev 1070 1071 def display_rev(self, rev): 1072 """Return a string representation of a revision in the repos for 1073 displaying to the user. 1074 1075 This can be a shortened revision string, e.g. for repositories 1076 using long hashes. 1077 1078 :raise NoSuchChangeset: If the given `rev` isn't found. 1079 :since 1.2: Always returns a string or `None`. 1080 """ 1081 norm_rev = self.normalize_rev(rev) 1082 return str(norm_rev) if norm_rev is not None else norm_rev 1083 1084 @abstractmethod 1085 def get_changes(self, old_path, old_rev, new_path, new_rev, 1086 ignore_ancestry=1): 1087 """Generates changes corresponding to generalized diffs. 1088 1089 Generator that yields change tuples (old_node, new_node, kind, change) 1090 for each node change between the two arbitrary (path,rev) pairs. 1091 1092 The old_node is assumed to be None when the change is an ADD, 1093 the new_node is assumed to be None when the change is a DELETE. 1094 """ 1095 pass 1096 1097 def is_viewable(self, perm): 1098 """Return True if view permission is granted on the repository.""" 1099 return 'BROWSER_VIEW' in perm(self.resource.child('source', '/')) 1100 1101 can_view = is_viewable # 0.12 compatibility 1102 1103 1104class Node(object, metaclass=ABCMeta): 1105 """Represents a directory or file in the repository at a given revision.""" 1106 1107 DIRECTORY = "dir" 1108 FILE = "file" 1109 1110 realm = RepositoryManager.source_realm 1111 1112 @property 1113 def resource(self): 1114 return Resource(self.realm, self.path, self.rev, self.repos.resource) 1115 1116 # created_path and created_rev properties refer to the Node "creation" 1117 # in the Subversion meaning of a Node in a versioned tree (see #3340). 1118 # 1119 # Those properties must be set by subclasses. 1120 # 1121 created_rev = None 1122 created_path = None 1123 1124 def __init__(self, repos, path, rev, kind): 1125 assert kind in (Node.DIRECTORY, Node.FILE), \ 1126 "Unknown node kind %s" % kind 1127 self.repos = repos 1128 self.path = to_unicode(path) 1129 self.rev = rev 1130 self.kind = kind 1131 1132 def __repr__(self): 1133 name = '%s:%s' % (self.repos.name, self.path) 1134 if self.rev is not None: 1135 name += '@' + str(self.rev) 1136 return '<%s %r>' % (self.__class__.__name__, name) 1137 1138 @abstractmethod 1139 def get_content(self): 1140 """Return a stream for reading the content of the node. 1141 1142 This method will return `None` for directories. 1143 The returned object must support a `read([len])` method. 1144 """ 1145 pass 1146 1147 def get_processed_content(self, keyword_substitution=True, eol_hint=None): 1148 """Return a stream for reading the content of the node, with some 1149 standard processing applied. 1150 1151 :param keyword_substitution: if `True`, meta-data keywords 1152 present in the content like ``$Rev$`` are substituted 1153 (which keyword are substituted and how they are 1154 substituted is backend specific) 1155 1156 :param eol_hint: which style of line ending is expected if 1157 `None` was explicitly specified for the file itself in 1158 the version control backend (for example in Subversion, 1159 if it was set to ``'native'``). It can be `None`, 1160 ``'LF'``, ``'CR'`` or ``'CRLF'``. 1161 """ 1162 return self.get_content() 1163 1164 @abstractmethod 1165 def get_entries(self): 1166 """Generator that yields the immediate child entries of a directory. 1167 1168 The entries are returned in no particular order. 1169 If the node is a file, this method returns `None`. 1170 """ 1171 pass 1172 1173 @abstractmethod 1174 def get_history(self, limit=None): 1175 """Provide backward history for this Node. 1176 1177 Generator that yields `(path, rev, chg)` tuples, one for each revision 1178 in which the node was changed. This generator will follow copies and 1179 moves of a node (if the underlying version control system supports 1180 that), which will be indicated by the first element of the tuple 1181 (i.e. the path) changing. 1182 Starts with an entry for the current revision. 1183 1184 :param limit: if given, yield at most ``limit`` results. 1185 """ 1186 pass 1187 1188 def get_previous(self): 1189 """Return the change event corresponding to the previous revision. 1190 1191 This returns a `(path, rev, chg)` tuple. 1192 """ 1193 skip = True 1194 for p in self.get_history(2): 1195 if skip: 1196 skip = False 1197 else: 1198 return p 1199 1200 @abstractmethod 1201 def get_annotations(self): 1202 """Provide detailed backward history for the content of this Node. 1203 1204 Retrieve an array of revisions, one `rev` for each line of content 1205 for that node. 1206 Only expected to work on (text) FILE nodes, of course. 1207 """ 1208 pass 1209 1210 @abstractmethod 1211 def get_properties(self): 1212 """Returns the properties (meta-data) of the node, as a dictionary. 1213 1214 The set of properties depends on the version control system. 1215 """ 1216 pass 1217 1218 @abstractmethod 1219 def get_content_length(self): 1220 """The length in bytes of the content. 1221 1222 Will be `None` for a directory. 1223 """ 1224 pass 1225 content_length = property(lambda self: self.get_content_length()) 1226 1227 @abstractmethod 1228 def get_content_type(self): 1229 """The MIME type corresponding to the content, if known. 1230 1231 Will be `None` for a directory. 1232 """ 1233 pass 1234 content_type = property(lambda self: self.get_content_type()) 1235 1236 def get_name(self): 1237 return self.path.split('/')[-1] 1238 name = property(lambda self: self.get_name()) 1239 1240 @abstractmethod 1241 def get_last_modified(self): 1242 pass 1243 last_modified = property(lambda self: self.get_last_modified()) 1244 1245 isdir = property(lambda self: self.kind == Node.DIRECTORY) 1246 isfile = property(lambda self: self.kind == Node.FILE) 1247 1248 def is_viewable(self, perm): 1249 """Return True if view permission is granted on the node.""" 1250 return ('BROWSER_VIEW' if self.isdir else 'FILE_VIEW') \ 1251 in perm(self.resource) 1252 1253 can_view = is_viewable # 0.12 compatibility 1254 1255 1256class Changeset(object, metaclass=ABCMeta): 1257 """Represents a set of changes committed at once in a repository.""" 1258 1259 ADD = 'add' 1260 COPY = 'copy' 1261 DELETE = 'delete' 1262 EDIT = 'edit' 1263 MOVE = 'move' 1264 1265 # change types which can have diff associated to them 1266 DIFF_CHANGES = (EDIT, COPY, MOVE) # MERGE 1267 OTHER_CHANGES = (ADD, DELETE) 1268 ALL_CHANGES = DIFF_CHANGES + OTHER_CHANGES 1269 1270 realm = RepositoryManager.changeset_realm 1271 1272 @property 1273 def resource(self): 1274 return Resource(self.realm, self.rev, parent=self.repos.resource) 1275 1276 def __init__(self, repos, rev, message, author, date): 1277 self.repos = repos 1278 self.rev = rev 1279 self.message = message or '' 1280 self.author = author or '' 1281 self.date = date 1282 1283 def __repr__(self): 1284 name = '%s@%s' % (self.repos.name, self.rev) 1285 return '<%s %r>' % (self.__class__.__name__, name) 1286 1287 def get_properties(self): 1288 """Returns the properties (meta-data) of the node, as a dictionary. 1289 1290 The set of properties depends on the version control system. 1291 1292 Warning: this used to yield 4-elements tuple (besides `name` and 1293 `text`, there were `wikiflag` and `htmlclass` values). 1294 This is now replaced by the usage of IPropertyRenderer (see #1601). 1295 """ 1296 return [] 1297 1298 @abstractmethod 1299 def get_changes(self): 1300 """Generator that produces a tuple for every change in the changeset. 1301 1302 The tuple will contain `(path, kind, change, base_path, base_rev)`, 1303 where `change` can be one of Changeset.ADD, Changeset.COPY, 1304 Changeset.DELETE, Changeset.EDIT or Changeset.MOVE, 1305 and `kind` is one of Node.FILE or Node.DIRECTORY. 1306 The `path` is the targeted path for the `change` (which is 1307 the ''deleted'' path for a DELETE change). 1308 The `base_path` and `base_rev` are the source path and rev for the 1309 action (`None` and `-1` in the case of an ADD change). 1310 """ 1311 pass 1312 1313 def get_branches(self): 1314 """Yield branches to which this changeset belong. 1315 Each branch is given as a pair `(name, head)`, where `name` is 1316 the branch name and `head` a flag set if the changeset is a head 1317 for this branch (i.e. if it has no children changeset). 1318 """ 1319 return [] 1320 1321 def get_tags(self): 1322 """Yield tags associated with this changeset. 1323 1324 .. versionadded :: 1.0 1325 """ 1326 return [] 1327 1328 def get_bookmarks(self): 1329 """Yield bookmarks associated with this changeset. 1330 1331 .. versionadded :: 1.1.5 1332 """ 1333 return [] 1334 1335 def is_viewable(self, perm): 1336 """Return True if view permission is granted on the changeset.""" 1337 return 'CHANGESET_VIEW' in perm(self.resource) 1338 1339 can_view = is_viewable # 0.12 compatibility 1340 1341 1342class EmptyChangeset(Changeset): 1343 """Changeset that contains no changes. This is typically used when the 1344 changeset can't be retrieved.""" 1345 1346 def __init__(self, repos, rev, message=None, author=None, date=None): 1347 if date is None: 1348 date = datetime(1970, 1, 1, tzinfo=utc) 1349 super().__init__(repos, rev, message, author, date) 1350 1351 def get_changes(self): 1352 return iter([]) 1353 1354 1355# Note: Since Trac 0.12, Exception PermissionDenied class is gone, 1356# and class Authorizer is gone as well. 1357# 1358# Fine-grained permissions are now handled via normal permission policies. 1359