1# -*- coding: utf-8 -*- 2 3# Copyright(C) 2010-2014 Romain Bignon, Laurent Bachelier 4# 5# This file is part of weboob. 6# 7# weboob is free software: you can redistribute it and/or modify 8# it under the terms of the GNU Lesser General Public License as published by 9# the Free Software Foundation, either version 3 of the License, or 10# (at your option) any later version. 11# 12# weboob is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU Lesser General Public License for more details. 16# 17# You should have received a copy of the GNU Lesser General Public License 18# along with weboob. If not, see <http://www.gnu.org/licenses/>. 19 20 21from __future__ import print_function 22import imp 23import posixpath 24import shutil 25import re 26import sys 27import os 28import subprocess 29import hashlib 30from compileall import compile_dir 31from contextlib import closing, contextmanager 32from datetime import datetime 33from io import BytesIO, StringIO 34from tempfile import NamedTemporaryFile 35 36from weboob.exceptions import BrowserHTTPError, BrowserHTTPNotFound, ModuleInstallError 37from .modules import LoadedModule 38from weboob.tools.log import getLogger 39from weboob.tools.misc import get_backtrace, to_unicode, find_exe 40from weboob.tools.compat import basestring, unicode 41try: 42 from ConfigParser import RawConfigParser, DEFAULTSECT 43except ImportError: 44 from configparser import RawConfigParser, DEFAULTSECT 45 46 47@contextmanager 48def open_for_config(filename): 49 if sys.version_info.major == 2: 50 f = NamedTemporaryFile(mode='wb', 51 dir=os.path.dirname(filename), 52 delete=False) 53 else: 54 f = NamedTemporaryFile(mode='w', encoding='utf-8', 55 dir=os.path.dirname(filename), 56 delete=False) 57 yield f 58 os.rename(f.name, filename) 59 60 61class ModuleInfo(object): 62 """ 63 Information about a module available on a repository. 64 """ 65 66 def __init__(self, name): 67 self.name = name 68 69 # path to the local directory containing this module. 70 self.path = None 71 self.url = None 72 self.repo_url = None 73 74 self.version = 0 75 self.capabilities = () 76 self.description = u'' 77 self.maintainer = u'' 78 self.license = u'' 79 self.icon = u'' 80 self.urls = u'' 81 82 def load(self, items): 83 self.version = int(items['version']) 84 self.capabilities = items['capabilities'].split() 85 self.description = to_unicode(items['description']) 86 self.maintainer = to_unicode(items['maintainer']) 87 self.license = to_unicode(items['license']) 88 self.icon = items['icon'].strip() or None 89 self.urls = items['urls'] 90 91 def has_caps(self, *caps): 92 """Return True if module implements at least one of the caps.""" 93 if len(caps) == 1 and isinstance(caps[0], (list, tuple)): 94 caps = caps[0] 95 for c in caps: 96 if type(c) == type: 97 c = c.__name__ 98 if c in self.capabilities: 99 return True 100 return False 101 102 def is_installed(self): 103 return self.path is not None 104 105 def is_local(self): 106 return self.url is None 107 108 def dump(self): 109 return (('version', self.version), 110 ('capabilities', ' '.join(self.capabilities)), 111 ('description', self.description), 112 ('maintainer', self.maintainer), 113 ('license', self.license), 114 ('icon', self.icon or ''), 115 ('urls', self.urls), 116 ) 117 118 119class RepositoryUnavailable(Exception): 120 """ 121 Repository in not available. 122 """ 123 124 125class Repository(object): 126 """ 127 Represents a repository. 128 """ 129 INDEX = 'modules.list' 130 KEYDIR = '.keys' 131 KEYRING = 'trusted.gpg' 132 133 def __init__(self, url): 134 self.url = url 135 self.name = u'' 136 self.update = 0 137 self.maintainer = u'' 138 self.local = None 139 self.signed = False 140 self.key_update = 0 141 self.obsolete = False 142 self.logger = getLogger('repository') 143 self.errors = {} 144 145 self.modules = {} 146 147 if self.url.startswith('file://'): 148 self.local = True 149 elif re.match('https?://.*', self.url): 150 self.local = False 151 else: 152 # This is probably a file in ~/.weboob/repositories/, we 153 # don't know if this is a local or a remote repository. 154 with open(self.url, 'r') as fp: 155 self.parse_index(fp) 156 157 def __repr__(self): 158 return '<Repository %r>' % self.name 159 160 def localurl2path(self): 161 """ 162 Get a local path of a file:// URL. 163 """ 164 assert self.local is True 165 166 if self.url.startswith('file://'): 167 return self.url[len('file://'):] 168 return self.url 169 170 def retrieve_index(self, browser, repo_path): 171 """ 172 Retrieve the index file of this repository. It can use network 173 if this is a remote repository. 174 175 :param repo_path: path to save the downloaded index file (if any). 176 :type repo_path: str or None 177 """ 178 built = False 179 if self.local: 180 # Repository is local, open the file. 181 filename = os.path.join(self.localurl2path(), self.INDEX) 182 try: 183 fp = open(filename, 'r') 184 except IOError: 185 # This local repository doesn't contain a built modules.list index. 186 self.name = Repositories.url2filename(self.url) 187 self.build_index(self.localurl2path(), filename) 188 built = True 189 fp = open(filename, 'r') 190 else: 191 # This is a remote repository, download file 192 try: 193 fp = StringIO(browser.open(posixpath.join(self.url, self.INDEX)).text) 194 except BrowserHTTPError as e: 195 raise RepositoryUnavailable(unicode(e)) 196 197 self.parse_index(fp) 198 199 # this value can be changed by parse_index 200 if self.local and not built: 201 # Always rebuild index of a local repository. 202 self.build_index(self.localurl2path(), filename) 203 204 # Save the repository index in ~/.weboob/repositories/ 205 if repo_path: 206 self.save(repo_path, private=True) 207 208 def retrieve_keyring(self, browser, keyring_path, progress): 209 # ignore local 210 if self.local: 211 return 212 213 keyring = Keyring(keyring_path) 214 # prevent previously signed repos from going unsigned 215 if not self.signed and keyring.exists(): 216 raise RepositoryUnavailable('Previously signed repository can not go unsigned') 217 if not self.signed: 218 return 219 220 if not keyring.exists() or self.key_update > keyring.version: 221 # This is a remote repository, download file 222 try: 223 keyring_data = browser.open(posixpath.join(self.url, self.KEYRING)).content 224 sig_data = browser.open(posixpath.join(self.url, self.KEYRING + '.sig')).content 225 except BrowserHTTPError as e: 226 raise RepositoryUnavailable(unicode(e)) 227 if keyring.exists(): 228 if not keyring.is_valid(keyring_data, sig_data): 229 raise InvalidSignature('the keyring itself') 230 progress.progress(0.0, 'The keyring was updated (and validated by the previous one).') 231 elif not progress.prompt('The repository %s isn\'t trusted yet.\nFingerprint of keyring is %s\nAre you sure you want to continue?' % (self.url, hashlib.sha1(keyring_data).hexdigest())): 232 raise RepositoryUnavailable('Repository not trusted') 233 keyring.save(keyring_data, self.key_update) 234 progress.progress(0.0, str(keyring)) 235 236 def parse_index(self, fp): 237 """ 238 Parse index of a repository 239 240 :param fp: file descriptor to read 241 :type fp: buffer 242 """ 243 config = RawConfigParser() 244 config.readfp(fp) 245 246 # Read default parameters 247 items = dict(config.items(DEFAULTSECT)) 248 try: 249 self.name = items['name'] 250 self.update = int(items['update']) 251 self.maintainer = items['maintainer'] 252 self.signed = bool(int(items.get('signed', '0'))) 253 self.key_update = int(items.get('key_update', '0')) 254 self.obsolete = bool(int(items.get('obsolete', '0'))) 255 except KeyError as e: 256 raise RepositoryUnavailable('Missing global parameters in repository: %s' % e) 257 except ValueError as e: 258 raise RepositoryUnavailable('Incorrect value in repository parameters: %s' % e) 259 260 if len(self.name) == 0: 261 raise RepositoryUnavailable('Name is empty') 262 263 if 'url' in items: 264 self.url = items['url'] 265 self.local = self.url.startswith('file://') 266 elif self.local is None: 267 raise RepositoryUnavailable('Missing "url" key in settings') 268 269 # Load modules 270 self.modules.clear() 271 for section in config.sections(): 272 module = ModuleInfo(section) 273 module.load(dict(config.items(section))) 274 if not self.local: 275 module.url = posixpath.join(self.url, '%s.tar.gz' % module.name) 276 module.repo_url = self.url 277 module.signed = self.signed 278 self.modules[section] = module 279 280 def build_index(self, path, filename): 281 """ 282 Rebuild index of modules of repository. 283 284 :param path: path of the repository 285 :type path: str 286 :param filename: file to save index 287 :type filename: str 288 """ 289 self.logger.debug('Rebuild index') 290 self.modules.clear() 291 self.errors.clear() 292 293 if os.path.isdir(os.path.join(path, self.KEYDIR)): 294 self.signed = True 295 self.key_update = self.get_tree_mtime(os.path.join(path, self.KEYDIR), True) 296 else: 297 self.signed = False 298 self.key_update = 0 299 300 for name in sorted(os.listdir(path)): 301 module_path = os.path.join(path, name) 302 if not os.path.isdir(module_path) or '.' in name or name == self.KEYDIR or not os.path.exists(os.path.join(module_path, '__init__.py')): 303 continue 304 305 try: 306 fp, pathname, description = imp.find_module(name, [path]) 307 try: 308 module = LoadedModule(imp.load_module(name, fp, pathname, description)) 309 finally: 310 if fp: 311 fp.close() 312 except Exception as e: 313 self.logger.warning('Unable to build module %s: [%s] %s' % (name, type(e).__name__, e)) 314 bt = get_backtrace(e) 315 self.logger.debug(bt) 316 self.errors[name] = bt 317 else: 318 m = ModuleInfo(module.name) 319 m.version = self.get_tree_mtime(module_path) 320 m.capabilities = list(set([c.__name__ for c in module.iter_caps()])) 321 m.description = module.description 322 m.maintainer = module.maintainer 323 m.license = module.license 324 m.icon = module.icon or '' 325 self.modules[module.name] = m 326 327 self.update = int(datetime.now().strftime('%Y%m%d%H%M')) 328 self.save(filename) 329 330 @staticmethod 331 def get_tree_mtime(path, include_root=False): 332 mtime = 0 333 if include_root: 334 mtime = int(datetime.fromtimestamp(os.path.getmtime(path)).strftime('%Y%m%d%H%M')) 335 for root, dirs, files in os.walk(path): 336 for f in files: 337 if f.endswith('.pyc'): 338 continue 339 m = int(datetime.fromtimestamp(os.path.getmtime(os.path.join(root, f))).strftime('%Y%m%d%H%M')) 340 mtime = max(mtime, m) 341 342 return mtime 343 344 def save(self, filename, private=False): 345 """ 346 Save repository into a file (modules.list for example). 347 348 :param filename: path to file to save repository. 349 :type filename: str 350 :param private: if enabled, save URL of repository. 351 :type private: bool 352 """ 353 config = RawConfigParser() 354 config.set(DEFAULTSECT, 'name', self.name) 355 config.set(DEFAULTSECT, 'update', self.update) 356 config.set(DEFAULTSECT, 'maintainer', self.maintainer) 357 config.set(DEFAULTSECT, 'signed', int(self.signed)) 358 config.set(DEFAULTSECT, 'key_update', self.key_update) 359 if private: 360 config.set(DEFAULTSECT, 'url', self.url) 361 362 for module in self.modules.values(): 363 config.add_section(module.name) 364 for key, value in module.dump(): 365 if sys.version_info.major == 2: 366 # python2's configparser enforces bytes coercion with str(value)... 367 config.set(module.name, key, to_unicode(value).encode('utf-8')) 368 else: 369 config.set(module.name, key, value) 370 371 with open_for_config(filename) as f: 372 config.write(f) 373 374 375class Versions(object): 376 VERSIONS_LIST = 'versions.list' 377 378 def __init__(self, path): 379 self.path = path 380 self.versions = {} 381 382 try: 383 with open(os.path.join(self.path, self.VERSIONS_LIST), 'r') as fp: 384 config = RawConfigParser() 385 config.readfp(fp) 386 387 # Read default parameters 388 for key, value in config.items(DEFAULTSECT): 389 self.versions[key] = int(value) 390 except IOError: 391 pass 392 393 def get(self, name): 394 return self.versions.get(name, None) 395 396 def set(self, name, version): 397 self.versions[name] = int(version) 398 self.save() 399 400 def save(self): 401 config = RawConfigParser() 402 for name, version in self.versions.items(): 403 config.set(DEFAULTSECT, name, version) 404 405 with open_for_config(os.path.join(self.path, self.VERSIONS_LIST)) as fp: 406 config.write(fp) 407 408 409class IProgress(object): 410 def progress(self, percent, message): 411 raise NotImplementedError() 412 413 def error(self, message): 414 raise NotImplementedError() 415 416 def prompt(self, message): 417 raise NotImplementedError() 418 419 def __repr__(self): 420 return '<%s>' % self.__class__.__name__ 421 422 423class PrintProgress(IProgress): 424 def progress(self, percent, message): 425 print('=== [%3.0f%%] %s' % (percent*100, message), file=sys.stderr) 426 427 def error(self, message): 428 print('ERROR: %s' % message, file=sys.stderr) 429 430 def prompt(self, message): 431 print('%s (Y/n): *** ASSUMING YES ***' % message, file=sys.stderr) 432 return True 433 434 435DEFAULT_SOURCES_LIST = \ 436"""# List of Weboob repositories 437# 438# The entries below override the entries above (with 439# backends of the same name). 440 441https://updates.weboob.org/%(version)s/main/ 442 443# DEVELOPMENT 444# If you want to hack on Weboob modules, you may add a 445# reference to sources, for example: 446#file:///home/rom1/src/weboob/modules/ 447""" 448 449 450class Repositories(object): 451 SOURCES_LIST = 'sources.list' 452 MODULES_DIR = 'modules' 453 REPOS_DIR = 'repositories' 454 KEYRINGS_DIR = 'keyrings' 455 ICONS_DIR = 'icons' 456 457 SHARE_DIRS = [MODULES_DIR, REPOS_DIR, KEYRINGS_DIR, ICONS_DIR] 458 459 def __init__(self, workdir, datadir, version): 460 self.logger = getLogger('repositories') 461 self.version = version 462 463 self.browser = None 464 465 self.workdir = workdir 466 self.datadir = datadir 467 self.sources_list = os.path.join(self.workdir, self.SOURCES_LIST) 468 self.modules_dir = os.path.join(self.datadir, self.MODULES_DIR, self.version) 469 self.repos_dir = os.path.join(self.datadir, self.REPOS_DIR) 470 self.keyrings_dir = os.path.join(self.datadir, self.KEYRINGS_DIR) 471 self.icons_dir = os.path.join(self.datadir, self.ICONS_DIR) 472 473 self.create_dir(self.datadir) 474 self.create_dir(self.modules_dir) 475 self.create_dir(self.repos_dir) 476 self.create_dir(self.keyrings_dir) 477 self.create_dir(self.icons_dir) 478 479 self.versions = Versions(self.modules_dir) 480 481 self.repositories = [] 482 483 if not os.path.exists(self.sources_list): 484 with open_for_config(self.sources_list) as f: 485 f.write(DEFAULT_SOURCES_LIST) 486 self.update() 487 else: 488 self.load() 489 490 def load_browser(self): 491 from weboob.browser.browsers import Browser 492 from weboob.browser.profiles import Weboob as WeboobProfile 493 from weboob.tools.compat import getproxies 494 495 class WeboobBrowser(Browser): 496 PROFILE = WeboobProfile(self.version) 497 if self.browser is None: 498 self.browser = WeboobBrowser( 499 logger=getLogger('browser', parent=self.logger), 500 proxy=getproxies()) 501 502 def create_dir(self, name): 503 if not os.path.exists(name): 504 os.makedirs(name) 505 elif not os.path.isdir(name): 506 self.logger.error(u'"%s" is not a directory' % name) 507 508 def _extend_module_info(self, repo, info): 509 if repo.local: 510 info.path = repo.localurl2path() 511 elif self.versions.get(info.name) is not None: 512 info.path = self.modules_dir 513 514 return info 515 516 def get_all_modules_info(self, caps=None): 517 """ 518 Get all ModuleInfo instances available. 519 520 :param caps: filter on capabilities: 521 :type caps: list[str] 522 :rtype: dict[:class:`ModuleInfo`] 523 """ 524 modules = {} 525 for repos in reversed(self.repositories): 526 for name, info in repos.modules.items(): 527 if name not in modules and (not caps or info.has_caps(caps)): 528 modules[name] = self._extend_module_info(repos, info) 529 return modules 530 531 def get_module_info(self, name): 532 """ 533 Get ModuleInfo object of a module. 534 535 It tries all repositories from last to first, and set 536 the 'path' attribute of ModuleInfo if it is installed. 537 """ 538 for repos in reversed(self.repositories): 539 if name in repos.modules: 540 m = repos.modules[name] 541 self._extend_module_info(repos, m) 542 return m 543 return None 544 545 def load(self): 546 """ 547 Load repositories from ~/.local/share/weboob/repositories/. 548 """ 549 self.repositories = [] 550 for name in sorted(os.listdir(self.repos_dir)): 551 path = os.path.join(self.repos_dir, name) 552 try: 553 repository = Repository(path) 554 self.repositories.append(repository) 555 except RepositoryUnavailable as e: 556 print('Unable to load repository %s (%s), try to update repositories.' % (name, e), file=sys.stderr) 557 558 def get_module_icon_path(self, module): 559 return os.path.join(self.icons_dir, '%s.png' % module.name) 560 561 def retrieve_icon(self, module): 562 """ 563 Retrieve the icon of a module and save it in ~/.local/share/weboob/icons/. 564 """ 565 self.load_browser() 566 if not isinstance(module, ModuleInfo): 567 module = self.get_module_info(module) 568 569 dest_path = self.get_module_icon_path(module) 570 571 icon_url = module.icon 572 if not icon_url: 573 if module.is_local(): 574 icon_path = os.path.join(module.path, module.name, 'favicon.png') 575 if module.path and os.path.exists(icon_path): 576 shutil.copy(icon_path, dest_path) 577 return 578 else: 579 icon_url = module.url.replace('.tar.gz', '.png') 580 581 try: 582 icon = self.browser.open(icon_url) 583 except BrowserHTTPNotFound: 584 pass # no icon, no problem 585 else: 586 with open(dest_path, 'wb') as fp: 587 fp.write(icon.content) 588 589 def _parse_source_list(self): 590 l = [] 591 with open(self.sources_list, 'r') as f: 592 for line in f: 593 line = line.strip() % {'version': self.version} 594 m = re.match('(file|https?)://.*', line) 595 if m: 596 l.append(line) 597 return l 598 599 def update_repositories(self, progress=PrintProgress()): 600 self.load_browser() 601 """ 602 Update list of repositories by downloading them 603 and put them in ~/.local/share/weboob/repositories/. 604 605 :param progress: observer object. 606 :type progress: :class:`IProgress` 607 """ 608 self.repositories = [] 609 for name in os.listdir(self.repos_dir): 610 os.remove(os.path.join(self.repos_dir, name)) 611 612 gpg_found = Keyring.find_gpg() or Keyring.find_gpgv() 613 for line in self._parse_source_list(): 614 progress.progress(0.0, 'Getting %s' % line) 615 repository = Repository(line) 616 filename = self.url2filename(repository.url) 617 prio_filename = '%02d-%s' % (len(self.repositories), filename) 618 repo_path = os.path.join(self.repos_dir, prio_filename) 619 keyring_path = os.path.join(self.keyrings_dir, filename) 620 try: 621 repository.retrieve_index(self.browser, repo_path) 622 if gpg_found: 623 repository.retrieve_keyring(self.browser, keyring_path, progress) 624 else: 625 progress.error('Cannot find gpg or gpgv to check for repository authenticity.\n' 626 'You should install GPG for better security.') 627 except RepositoryUnavailable as e: 628 progress.error('Unable to load repository: %s' % e) 629 else: 630 self.repositories.append(repository) 631 if repository.obsolete: 632 last_update = datetime.strptime(str(repository.update), '%Y%m%d%H%M').strftime('%Y-%m-%d') 633 progress.error('This repository does not receive updates anymore (since %s).\n' 634 'Your weboob version is probably obsolete and should be upgraded.' % last_update) 635 636 def check_repositories(self): 637 """ 638 Check if sources.list is consistent with repositories 639 """ 640 l = [] 641 for line in self._parse_source_list(): 642 repository = Repository(line) 643 filename = self.url2filename(repository.url) 644 prio_filename = '%02d-%s' % (len(l), filename) 645 repo_path = os.path.join(self.repos_dir, prio_filename) 646 if not os.path.isfile(repo_path): 647 return False 648 l.append(repository) 649 return True 650 651 def update(self, progress=PrintProgress()): 652 """ 653 Update repositories and install new packages versions. 654 655 :param progress: observer object. 656 :type progress: :class:`IProgress` 657 """ 658 self.update_repositories(progress) 659 660 to_update = [] 661 for name, info in self.get_all_modules_info().items(): 662 if not info.is_local() and info.is_installed(): 663 if self.versions.get(name) != info.version: 664 to_update.append(info) 665 666 if len(to_update) == 0: 667 progress.progress(1.0, 'All modules are up-to-date.') 668 return 669 670 class InstallProgress(PrintProgress): 671 def __init__(self, n): 672 self.n = n 673 674 def progress(self, percent, message): 675 progress.progress(float(self.n)/len(to_update) + 1.0/len(to_update)*percent, message) 676 677 for n, info in enumerate(to_update): 678 inst_progress = InstallProgress(n) 679 try: 680 self.install(info, inst_progress) 681 except ModuleInstallError as e: 682 inst_progress.progress(1.0, unicode(e)) 683 684 def install(self, module, progress=PrintProgress()): 685 """ 686 Install a module. 687 688 :param module: module to install 689 :type module: :class:`str` or :class:`ModuleInfo` 690 :param progress: observer object 691 :type progress: :class:`IProgress` 692 """ 693 import tarfile 694 self.load_browser() 695 696 if isinstance(module, ModuleInfo): 697 info = module 698 elif isinstance(module, basestring): 699 progress.progress(0.0, 'Looking for module %s' % module) 700 info = self.get_module_info(module) 701 if not info: 702 raise ModuleInstallError('Module "%s" does not exist' % module) 703 else: 704 raise ValueError('"module" parameter might be a ModuleInfo object or a string, not %r' % module) 705 706 module = info 707 708 if module.is_local(): 709 raise ModuleInstallError('%s is available on local.' % module.name) 710 711 module_dir = os.path.join(self.modules_dir, module.name) 712 installed = self.versions.get(module.name) 713 if installed is None or not os.path.exists(module_dir): 714 progress.progress(0.2, 'Module %s is not installed yet' % module.name) 715 elif module.version > installed: 716 progress.progress(0.2, 'A new version of %s is available' % module.name) 717 else: 718 raise ModuleInstallError('The latest version of %s is already installed' % module.name) 719 720 progress.progress(0.3, 'Downloading module...') 721 try: 722 tardata = self.browser.open(module.url).content 723 except BrowserHTTPError as e: 724 raise ModuleInstallError('Unable to fetch module: %s' % e) 725 726 # Check signature 727 if module.signed and (Keyring.find_gpg() or Keyring.find_gpgv()): 728 progress.progress(0.5, 'Checking module authenticity...') 729 sig_data = self.browser.open(posixpath.join(module.url + '.sig')).content 730 keyring_path = os.path.join(self.keyrings_dir, self.url2filename(module.repo_url)) 731 keyring = Keyring(keyring_path) 732 if not keyring.exists(): 733 raise ModuleInstallError('No keyring found, please update repos.') 734 if not keyring.is_valid(tardata, sig_data): 735 raise ModuleInstallError('Invalid signature for %s.' % module.name) 736 737 # Extract module from tarball. 738 if os.path.isdir(module_dir): 739 shutil.rmtree(module_dir) 740 progress.progress(0.7, 'Setting up module...') 741 with closing(tarfile.open('', 'r:gz', BytesIO(tardata))) as tar: 742 tar.extractall(self.modules_dir) 743 if not os.path.isdir(module_dir): 744 raise ModuleInstallError('The archive for %s looks invalid.' % module.name) 745 # Precompile 746 compile_dir(module_dir, quiet=True) 747 748 self.versions.set(module.name, module.version) 749 750 progress.progress(0.9, 'Downloading icon...') 751 self.retrieve_icon(module) 752 753 progress.progress(1.0, 'Module %s has been installed!' % module.name) 754 755 @staticmethod 756 def url2filename(url): 757 """ 758 Get a safe file name for an URL. 759 760 All non-alphanumeric characters are replaced by _. 761 """ 762 return ''.join([l if l.isalnum() else '_' for l in url]) 763 764 def __iter__(self): 765 for repository in self.repositories: 766 yield repository 767 768 @property 769 def errors(self): 770 errors = {} 771 for repository in self: 772 errors.update(repository.errors) 773 return errors 774 775 776class InvalidSignature(Exception): 777 def __init__(self, filename): 778 self.filename = filename 779 super(InvalidSignature, self).__init__('Invalid signature for %s' % filename) 780 781 782class Keyring(object): 783 EXTENSION = '.gpg' 784 785 def __init__(self, path): 786 self.path = path + self.EXTENSION 787 self.vpath = path + '.version' 788 self.version = 0 789 790 if self.exists(): 791 with open(self.vpath, 'r') as f: 792 self.version = int(f.read().strip()) 793 else: 794 if os.path.exists(self.path): 795 os.remove(self.path) 796 if os.path.exists(self.vpath): 797 os.remove(self.vpath) 798 799 def exists(self): 800 if not os.path.exists(self.vpath): 801 return False 802 if os.path.exists(self.path): 803 # Check the file is not empty. 804 # This is because there was a bug creating empty keyring files. 805 with open(self.path, 'rb') as fp: 806 if len(fp.read().strip()): 807 return True 808 return False 809 810 def save(self, keyring_data, version): 811 with open(self.path, 'wb') as fp: 812 fp.write(keyring_data) 813 self.version = version 814 with open_for_config(self.vpath) as fp: 815 fp.write(str(version)) 816 817 @staticmethod 818 def find_gpgv(): 819 return find_exe('gpgv2') or find_exe('gpgv') 820 821 @staticmethod 822 def find_gpg(): 823 return find_exe('gpg2') or find_exe('gpg') 824 825 def is_valid(self, data, sigdata): 826 """ 827 Check if the data is signed by an accepted key. 828 data and sigdata should be strings. 829 """ 830 gpg = self.find_gpg() 831 gpgv = self.find_gpgv() 832 833 if gpg: 834 from tempfile import mkdtemp 835 gpg_homedir = mkdtemp(prefix='weboob_gpg_') 836 verify_command = [gpg, '--verify', '--no-options', 837 '--no-default-keyring', '--quiet', 838 '--homedir', gpg_homedir] 839 elif gpgv: 840 verify_command = [gpgv] 841 842 from tempfile import NamedTemporaryFile 843 with NamedTemporaryFile(suffix='.sig', delete=False) as sigfile: 844 temp_filename = sigfile.name 845 return_code = None 846 out = '' 847 err = '' 848 try: 849 sigfile.write(sigdata) 850 sigfile.flush() # very important 851 sigfile.close() 852 assert isinstance(data, bytes) 853 # Yes, all of it is necessary 854 proc = subprocess.Popen(verify_command + [ 855 '--status-fd', '1', 856 '--keyring', os.path.realpath(self.path), 857 os.path.realpath(sigfile.name), 858 '-'], 859 stdin=subprocess.PIPE, 860 stdout=subprocess.PIPE, 861 stderr=subprocess.PIPE) 862 out, err = proc.communicate(data) 863 return_code = proc.returncode 864 finally: 865 os.unlink(temp_filename) 866 if gpg: 867 shutil.rmtree(gpg_homedir) 868 869 if return_code or b'GOODSIG' not in out or b'VALIDSIG' not in out: 870 print(out, err, file=sys.stderr) 871 return False 872 return True 873 874 def __str__(self): 875 if self.exists(): 876 with open(self.path, 'rb') as f: 877 h = hashlib.sha1(f.read()).hexdigest() 878 return 'Keyring version %s, checksum %s' % (self.version, h) 879 return 'NO KEYRING' 880