1# -*- coding: utf-8 -*-
2
3# Copyright(C) 2010-2014 Romain Bignon, Laurent Bachelier
4#
5# This file is part of weboob.
6#
7# weboob is free software: you can redistribute it and/or modify
8# it under the terms of the GNU Lesser General Public License as published by
9# the Free Software Foundation, either version 3 of the License, or
10# (at your option) any later version.
11#
12# weboob is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU Lesser General Public License for more details.
16#
17# You should have received a copy of the GNU Lesser General Public License
18# along with weboob. If not, see <http://www.gnu.org/licenses/>.
19
20
21from __future__ import print_function
22import imp
23import posixpath
24import shutil
25import re
26import sys
27import os
28import subprocess
29import hashlib
30from compileall import compile_dir
31from contextlib import closing, contextmanager
32from datetime import datetime
33from io import BytesIO, StringIO
34from tempfile import NamedTemporaryFile
35
36from weboob.exceptions import BrowserHTTPError, BrowserHTTPNotFound, ModuleInstallError
37from .modules import LoadedModule
38from weboob.tools.log import getLogger
39from weboob.tools.misc import get_backtrace, to_unicode, find_exe
40from weboob.tools.compat import basestring, unicode
41try:
42    from ConfigParser import RawConfigParser, DEFAULTSECT
43except ImportError:
44    from configparser import RawConfigParser, DEFAULTSECT
45
46
47@contextmanager
48def open_for_config(filename):
49    if sys.version_info.major == 2:
50        f = NamedTemporaryFile(mode='wb',
51                               dir=os.path.dirname(filename),
52                               delete=False)
53    else:
54        f = NamedTemporaryFile(mode='w', encoding='utf-8',
55                               dir=os.path.dirname(filename),
56                               delete=False)
57    yield f
58    os.rename(f.name, filename)
59
60
61class ModuleInfo(object):
62    """
63    Information about a module available on a repository.
64    """
65
66    def __init__(self, name):
67        self.name = name
68
69        # path to the local directory containing this module.
70        self.path = None
71        self.url = None
72        self.repo_url = None
73
74        self.version = 0
75        self.capabilities = ()
76        self.description = u''
77        self.maintainer = u''
78        self.license = u''
79        self.icon = u''
80        self.urls = u''
81
82    def load(self, items):
83        self.version = int(items['version'])
84        self.capabilities = items['capabilities'].split()
85        self.description = to_unicode(items['description'])
86        self.maintainer = to_unicode(items['maintainer'])
87        self.license = to_unicode(items['license'])
88        self.icon = items['icon'].strip() or None
89        self.urls = items['urls']
90
91    def has_caps(self, *caps):
92        """Return True if module implements at least one of the caps."""
93        if len(caps) == 1 and isinstance(caps[0], (list, tuple)):
94            caps = caps[0]
95        for c in caps:
96            if type(c) == type:
97                c = c.__name__
98            if c in self.capabilities:
99                return True
100        return False
101
102    def is_installed(self):
103        return self.path is not None
104
105    def is_local(self):
106        return self.url is None
107
108    def dump(self):
109        return (('version', self.version),
110                ('capabilities', ' '.join(self.capabilities)),
111                ('description', self.description),
112                ('maintainer', self.maintainer),
113                ('license', self.license),
114                ('icon', self.icon or ''),
115                ('urls', self.urls),
116               )
117
118
119class RepositoryUnavailable(Exception):
120    """
121    Repository in not available.
122    """
123
124
125class Repository(object):
126    """
127    Represents a repository.
128    """
129    INDEX = 'modules.list'
130    KEYDIR = '.keys'
131    KEYRING = 'trusted.gpg'
132
133    def __init__(self, url):
134        self.url = url
135        self.name = u''
136        self.update = 0
137        self.maintainer = u''
138        self.local = None
139        self.signed = False
140        self.key_update = 0
141        self.obsolete = False
142        self.logger = getLogger('repository')
143        self.errors = {}
144
145        self.modules = {}
146
147        if self.url.startswith('file://'):
148            self.local = True
149        elif re.match('https?://.*', self.url):
150            self.local = False
151        else:
152            # This is probably a file in ~/.weboob/repositories/, we
153            # don't know if this is a local or a remote repository.
154            with open(self.url, 'r') as fp:
155                self.parse_index(fp)
156
157    def __repr__(self):
158        return '<Repository %r>' % self.name
159
160    def localurl2path(self):
161        """
162        Get a local path of a file:// URL.
163        """
164        assert self.local is True
165
166        if self.url.startswith('file://'):
167            return self.url[len('file://'):]
168        return self.url
169
170    def retrieve_index(self, browser, repo_path):
171        """
172        Retrieve the index file of this repository. It can use network
173        if this is a remote repository.
174
175        :param repo_path: path to save the downloaded index file (if any).
176        :type repo_path: str or None
177        """
178        built = False
179        if self.local:
180            # Repository is local, open the file.
181            filename = os.path.join(self.localurl2path(), self.INDEX)
182            try:
183                fp = open(filename, 'r')
184            except IOError:
185                # This local repository doesn't contain a built modules.list index.
186                self.name = Repositories.url2filename(self.url)
187                self.build_index(self.localurl2path(), filename)
188                built = True
189                fp = open(filename, 'r')
190        else:
191            # This is a remote repository, download file
192            try:
193                fp = StringIO(browser.open(posixpath.join(self.url, self.INDEX)).text)
194            except BrowserHTTPError as e:
195                raise RepositoryUnavailable(unicode(e))
196
197        self.parse_index(fp)
198
199        # this value can be changed by parse_index
200        if self.local and not built:
201            # Always rebuild index of a local repository.
202            self.build_index(self.localurl2path(), filename)
203
204        # Save the repository index in ~/.weboob/repositories/
205        if repo_path:
206            self.save(repo_path, private=True)
207
208    def retrieve_keyring(self, browser, keyring_path, progress):
209        # ignore local
210        if self.local:
211            return
212
213        keyring = Keyring(keyring_path)
214        # prevent previously signed repos from going unsigned
215        if not self.signed and keyring.exists():
216            raise RepositoryUnavailable('Previously signed repository can not go unsigned')
217        if not self.signed:
218            return
219
220        if not keyring.exists() or self.key_update > keyring.version:
221            # This is a remote repository, download file
222            try:
223                keyring_data = browser.open(posixpath.join(self.url, self.KEYRING)).content
224                sig_data = browser.open(posixpath.join(self.url, self.KEYRING + '.sig')).content
225            except BrowserHTTPError as e:
226                raise RepositoryUnavailable(unicode(e))
227            if keyring.exists():
228                if not keyring.is_valid(keyring_data, sig_data):
229                    raise InvalidSignature('the keyring itself')
230                progress.progress(0.0, 'The keyring was updated (and validated by the previous one).')
231            elif not progress.prompt('The repository %s isn\'t trusted yet.\nFingerprint of keyring is %s\nAre you sure you want to continue?' % (self.url, hashlib.sha1(keyring_data).hexdigest())):
232                raise RepositoryUnavailable('Repository not trusted')
233            keyring.save(keyring_data, self.key_update)
234            progress.progress(0.0, str(keyring))
235
236    def parse_index(self, fp):
237        """
238        Parse index of a repository
239
240        :param fp: file descriptor to read
241        :type fp: buffer
242        """
243        config = RawConfigParser()
244        config.readfp(fp)
245
246        # Read default parameters
247        items = dict(config.items(DEFAULTSECT))
248        try:
249            self.name = items['name']
250            self.update = int(items['update'])
251            self.maintainer = items['maintainer']
252            self.signed = bool(int(items.get('signed', '0')))
253            self.key_update = int(items.get('key_update', '0'))
254            self.obsolete = bool(int(items.get('obsolete', '0')))
255        except KeyError as e:
256            raise RepositoryUnavailable('Missing global parameters in repository: %s' % e)
257        except ValueError as e:
258            raise RepositoryUnavailable('Incorrect value in repository parameters: %s' % e)
259
260        if len(self.name) == 0:
261            raise RepositoryUnavailable('Name is empty')
262
263        if 'url' in items:
264            self.url = items['url']
265            self.local = self.url.startswith('file://')
266        elif self.local is None:
267            raise RepositoryUnavailable('Missing "url" key in settings')
268
269        # Load modules
270        self.modules.clear()
271        for section in config.sections():
272            module = ModuleInfo(section)
273            module.load(dict(config.items(section)))
274            if not self.local:
275                module.url = posixpath.join(self.url, '%s.tar.gz' % module.name)
276                module.repo_url = self.url
277                module.signed = self.signed
278            self.modules[section] = module
279
280    def build_index(self, path, filename):
281        """
282        Rebuild index of modules of repository.
283
284        :param path: path of the repository
285        :type path: str
286        :param filename: file to save index
287        :type filename: str
288        """
289        self.logger.debug('Rebuild index')
290        self.modules.clear()
291        self.errors.clear()
292
293        if os.path.isdir(os.path.join(path, self.KEYDIR)):
294            self.signed = True
295            self.key_update = self.get_tree_mtime(os.path.join(path, self.KEYDIR), True)
296        else:
297            self.signed = False
298            self.key_update = 0
299
300        for name in sorted(os.listdir(path)):
301            module_path = os.path.join(path, name)
302            if not os.path.isdir(module_path) or '.' in name or name == self.KEYDIR or not os.path.exists(os.path.join(module_path, '__init__.py')):
303                continue
304
305            try:
306                fp, pathname, description = imp.find_module(name, [path])
307                try:
308                    module = LoadedModule(imp.load_module(name, fp, pathname, description))
309                finally:
310                    if fp:
311                        fp.close()
312            except Exception as e:
313                self.logger.warning('Unable to build module %s: [%s] %s' % (name, type(e).__name__, e))
314                bt = get_backtrace(e)
315                self.logger.debug(bt)
316                self.errors[name] = bt
317            else:
318                m = ModuleInfo(module.name)
319                m.version = self.get_tree_mtime(module_path)
320                m.capabilities = list(set([c.__name__ for c in module.iter_caps()]))
321                m.description = module.description
322                m.maintainer = module.maintainer
323                m.license = module.license
324                m.icon = module.icon or ''
325                self.modules[module.name] = m
326
327        self.update = int(datetime.now().strftime('%Y%m%d%H%M'))
328        self.save(filename)
329
330    @staticmethod
331    def get_tree_mtime(path, include_root=False):
332        mtime = 0
333        if include_root:
334            mtime = int(datetime.fromtimestamp(os.path.getmtime(path)).strftime('%Y%m%d%H%M'))
335        for root, dirs, files in os.walk(path):
336            for f in files:
337                if f.endswith('.pyc'):
338                    continue
339                m = int(datetime.fromtimestamp(os.path.getmtime(os.path.join(root, f))).strftime('%Y%m%d%H%M'))
340                mtime = max(mtime, m)
341
342        return mtime
343
344    def save(self, filename, private=False):
345        """
346        Save repository into a file (modules.list for example).
347
348        :param filename: path to file to save repository.
349        :type filename: str
350        :param private: if enabled, save URL of repository.
351        :type private: bool
352        """
353        config = RawConfigParser()
354        config.set(DEFAULTSECT, 'name', self.name)
355        config.set(DEFAULTSECT, 'update', self.update)
356        config.set(DEFAULTSECT, 'maintainer', self.maintainer)
357        config.set(DEFAULTSECT, 'signed', int(self.signed))
358        config.set(DEFAULTSECT, 'key_update', self.key_update)
359        if private:
360            config.set(DEFAULTSECT, 'url', self.url)
361
362        for module in self.modules.values():
363            config.add_section(module.name)
364            for key, value in module.dump():
365                if sys.version_info.major == 2:
366                    # python2's configparser enforces bytes coercion with str(value)...
367                    config.set(module.name, key, to_unicode(value).encode('utf-8'))
368                else:
369                    config.set(module.name, key, value)
370
371        with open_for_config(filename) as f:
372            config.write(f)
373
374
375class Versions(object):
376    VERSIONS_LIST = 'versions.list'
377
378    def __init__(self, path):
379        self.path = path
380        self.versions = {}
381
382        try:
383            with open(os.path.join(self.path, self.VERSIONS_LIST), 'r') as fp:
384                config = RawConfigParser()
385                config.readfp(fp)
386
387                # Read default parameters
388                for key, value in config.items(DEFAULTSECT):
389                    self.versions[key] = int(value)
390        except IOError:
391            pass
392
393    def get(self, name):
394        return self.versions.get(name, None)
395
396    def set(self, name, version):
397        self.versions[name] = int(version)
398        self.save()
399
400    def save(self):
401        config = RawConfigParser()
402        for name, version in self.versions.items():
403            config.set(DEFAULTSECT, name, version)
404
405        with open_for_config(os.path.join(self.path, self.VERSIONS_LIST)) as fp:
406            config.write(fp)
407
408
409class IProgress(object):
410    def progress(self, percent, message):
411        raise NotImplementedError()
412
413    def error(self, message):
414        raise NotImplementedError()
415
416    def prompt(self, message):
417        raise NotImplementedError()
418
419    def __repr__(self):
420        return '<%s>' % self.__class__.__name__
421
422
423class PrintProgress(IProgress):
424    def progress(self, percent, message):
425        print('=== [%3.0f%%] %s' % (percent*100, message), file=sys.stderr)
426
427    def error(self, message):
428        print('ERROR: %s' % message, file=sys.stderr)
429
430    def prompt(self, message):
431        print('%s (Y/n): *** ASSUMING YES ***' % message, file=sys.stderr)
432        return True
433
434
435DEFAULT_SOURCES_LIST = \
436"""# List of Weboob repositories
437#
438# The entries below override the entries above (with
439# backends of the same name).
440
441https://updates.weboob.org/%(version)s/main/
442
443# DEVELOPMENT
444# If you want to hack on Weboob modules, you may add a
445# reference to sources, for example:
446#file:///home/rom1/src/weboob/modules/
447"""
448
449
450class Repositories(object):
451    SOURCES_LIST = 'sources.list'
452    MODULES_DIR = 'modules'
453    REPOS_DIR = 'repositories'
454    KEYRINGS_DIR = 'keyrings'
455    ICONS_DIR = 'icons'
456
457    SHARE_DIRS = [MODULES_DIR, REPOS_DIR, KEYRINGS_DIR, ICONS_DIR]
458
459    def __init__(self, workdir, datadir, version):
460        self.logger = getLogger('repositories')
461        self.version = version
462
463        self.browser = None
464
465        self.workdir = workdir
466        self.datadir = datadir
467        self.sources_list = os.path.join(self.workdir, self.SOURCES_LIST)
468        self.modules_dir = os.path.join(self.datadir, self.MODULES_DIR, self.version)
469        self.repos_dir = os.path.join(self.datadir, self.REPOS_DIR)
470        self.keyrings_dir = os.path.join(self.datadir, self.KEYRINGS_DIR)
471        self.icons_dir = os.path.join(self.datadir, self.ICONS_DIR)
472
473        self.create_dir(self.datadir)
474        self.create_dir(self.modules_dir)
475        self.create_dir(self.repos_dir)
476        self.create_dir(self.keyrings_dir)
477        self.create_dir(self.icons_dir)
478
479        self.versions = Versions(self.modules_dir)
480
481        self.repositories = []
482
483        if not os.path.exists(self.sources_list):
484            with open_for_config(self.sources_list) as f:
485                f.write(DEFAULT_SOURCES_LIST)
486            self.update()
487        else:
488            self.load()
489
490    def load_browser(self):
491        from weboob.browser.browsers import Browser
492        from weboob.browser.profiles import Weboob as WeboobProfile
493        from weboob.tools.compat import getproxies
494
495        class WeboobBrowser(Browser):
496            PROFILE = WeboobProfile(self.version)
497        if self.browser is None:
498            self.browser = WeboobBrowser(
499                logger=getLogger('browser', parent=self.logger),
500                proxy=getproxies())
501
502    def create_dir(self, name):
503        if not os.path.exists(name):
504            os.makedirs(name)
505        elif not os.path.isdir(name):
506            self.logger.error(u'"%s" is not a directory' % name)
507
508    def _extend_module_info(self, repo, info):
509        if repo.local:
510            info.path = repo.localurl2path()
511        elif self.versions.get(info.name) is not None:
512            info.path = self.modules_dir
513
514        return info
515
516    def get_all_modules_info(self, caps=None):
517        """
518        Get all ModuleInfo instances available.
519
520        :param caps: filter on capabilities:
521        :type caps: list[str]
522        :rtype: dict[:class:`ModuleInfo`]
523        """
524        modules = {}
525        for repos in reversed(self.repositories):
526            for name, info in repos.modules.items():
527                if name not in modules and (not caps or info.has_caps(caps)):
528                    modules[name] = self._extend_module_info(repos, info)
529        return modules
530
531    def get_module_info(self, name):
532        """
533        Get ModuleInfo object of a module.
534
535        It tries all repositories from last to first, and set
536        the 'path' attribute of ModuleInfo if it is installed.
537        """
538        for repos in reversed(self.repositories):
539            if name in repos.modules:
540                m = repos.modules[name]
541                self._extend_module_info(repos, m)
542                return m
543        return None
544
545    def load(self):
546        """
547        Load repositories from ~/.local/share/weboob/repositories/.
548        """
549        self.repositories = []
550        for name in sorted(os.listdir(self.repos_dir)):
551            path = os.path.join(self.repos_dir, name)
552            try:
553                repository = Repository(path)
554                self.repositories.append(repository)
555            except RepositoryUnavailable as e:
556                print('Unable to load repository %s (%s), try to update repositories.' % (name, e), file=sys.stderr)
557
558    def get_module_icon_path(self, module):
559        return os.path.join(self.icons_dir, '%s.png' % module.name)
560
561    def retrieve_icon(self, module):
562        """
563        Retrieve the icon of a module and save it in ~/.local/share/weboob/icons/.
564        """
565        self.load_browser()
566        if not isinstance(module, ModuleInfo):
567            module = self.get_module_info(module)
568
569        dest_path = self.get_module_icon_path(module)
570
571        icon_url = module.icon
572        if not icon_url:
573            if module.is_local():
574                icon_path = os.path.join(module.path, module.name, 'favicon.png')
575                if module.path and os.path.exists(icon_path):
576                    shutil.copy(icon_path, dest_path)
577                return
578            else:
579                icon_url = module.url.replace('.tar.gz', '.png')
580
581        try:
582            icon = self.browser.open(icon_url)
583        except BrowserHTTPNotFound:
584            pass  # no icon, no problem
585        else:
586            with open(dest_path, 'wb') as fp:
587                fp.write(icon.content)
588
589    def _parse_source_list(self):
590        l = []
591        with open(self.sources_list, 'r') as f:
592            for line in f:
593                line = line.strip() % {'version': self.version}
594                m = re.match('(file|https?)://.*', line)
595                if m:
596                    l.append(line)
597        return l
598
599    def update_repositories(self, progress=PrintProgress()):
600        self.load_browser()
601        """
602        Update list of repositories by downloading them
603        and put them in ~/.local/share/weboob/repositories/.
604
605        :param progress: observer object.
606        :type progress: :class:`IProgress`
607        """
608        self.repositories = []
609        for name in os.listdir(self.repos_dir):
610            os.remove(os.path.join(self.repos_dir, name))
611
612        gpg_found = Keyring.find_gpg() or Keyring.find_gpgv()
613        for line in self._parse_source_list():
614            progress.progress(0.0, 'Getting %s' % line)
615            repository = Repository(line)
616            filename = self.url2filename(repository.url)
617            prio_filename = '%02d-%s' % (len(self.repositories), filename)
618            repo_path = os.path.join(self.repos_dir, prio_filename)
619            keyring_path = os.path.join(self.keyrings_dir, filename)
620            try:
621                repository.retrieve_index(self.browser, repo_path)
622                if gpg_found:
623                    repository.retrieve_keyring(self.browser, keyring_path, progress)
624                else:
625                    progress.error('Cannot find gpg or gpgv to check for repository authenticity.\n'
626                                   'You should install GPG for better security.')
627            except RepositoryUnavailable as e:
628                progress.error('Unable to load repository: %s' % e)
629            else:
630                self.repositories.append(repository)
631                if repository.obsolete:
632                    last_update = datetime.strptime(str(repository.update), '%Y%m%d%H%M').strftime('%Y-%m-%d')
633                    progress.error('This repository does not receive updates anymore (since %s).\n'
634                                   'Your weboob version is probably obsolete and should be upgraded.' % last_update)
635
636    def check_repositories(self):
637        """
638        Check if sources.list is consistent with repositories
639        """
640        l = []
641        for line in self._parse_source_list():
642            repository = Repository(line)
643            filename = self.url2filename(repository.url)
644            prio_filename = '%02d-%s' % (len(l), filename)
645            repo_path = os.path.join(self.repos_dir, prio_filename)
646            if not os.path.isfile(repo_path):
647                return False
648            l.append(repository)
649        return True
650
651    def update(self, progress=PrintProgress()):
652        """
653        Update repositories and install new packages versions.
654
655        :param progress: observer object.
656        :type progress: :class:`IProgress`
657        """
658        self.update_repositories(progress)
659
660        to_update = []
661        for name, info in self.get_all_modules_info().items():
662            if not info.is_local() and info.is_installed():
663                if self.versions.get(name) != info.version:
664                    to_update.append(info)
665
666        if len(to_update) == 0:
667            progress.progress(1.0, 'All modules are up-to-date.')
668            return
669
670        class InstallProgress(PrintProgress):
671            def __init__(self, n):
672                self.n = n
673
674            def progress(self, percent, message):
675                progress.progress(float(self.n)/len(to_update) + 1.0/len(to_update)*percent, message)
676
677        for n, info in enumerate(to_update):
678            inst_progress = InstallProgress(n)
679            try:
680                self.install(info, inst_progress)
681            except ModuleInstallError as e:
682                inst_progress.progress(1.0, unicode(e))
683
684    def install(self, module, progress=PrintProgress()):
685        """
686        Install a module.
687
688        :param module: module to install
689        :type module: :class:`str` or :class:`ModuleInfo`
690        :param progress: observer object
691        :type progress: :class:`IProgress`
692        """
693        import tarfile
694        self.load_browser()
695
696        if isinstance(module, ModuleInfo):
697            info = module
698        elif isinstance(module, basestring):
699            progress.progress(0.0, 'Looking for module %s' % module)
700            info = self.get_module_info(module)
701            if not info:
702                raise ModuleInstallError('Module "%s" does not exist' % module)
703        else:
704            raise ValueError('"module" parameter might be a ModuleInfo object or a string, not %r' % module)
705
706        module = info
707
708        if module.is_local():
709            raise ModuleInstallError('%s is available on local.' % module.name)
710
711        module_dir = os.path.join(self.modules_dir, module.name)
712        installed = self.versions.get(module.name)
713        if installed is None or not os.path.exists(module_dir):
714            progress.progress(0.2, 'Module %s is not installed yet' % module.name)
715        elif module.version > installed:
716            progress.progress(0.2, 'A new version of %s is available' % module.name)
717        else:
718            raise ModuleInstallError('The latest version of %s is already installed' % module.name)
719
720        progress.progress(0.3, 'Downloading module...')
721        try:
722            tardata = self.browser.open(module.url).content
723        except BrowserHTTPError as e:
724            raise ModuleInstallError('Unable to fetch module: %s' % e)
725
726        # Check signature
727        if module.signed and (Keyring.find_gpg() or Keyring.find_gpgv()):
728            progress.progress(0.5, 'Checking module authenticity...')
729            sig_data = self.browser.open(posixpath.join(module.url + '.sig')).content
730            keyring_path = os.path.join(self.keyrings_dir, self.url2filename(module.repo_url))
731            keyring = Keyring(keyring_path)
732            if not keyring.exists():
733                raise ModuleInstallError('No keyring found, please update repos.')
734            if not keyring.is_valid(tardata, sig_data):
735                raise ModuleInstallError('Invalid signature for %s.' % module.name)
736
737        # Extract module from tarball.
738        if os.path.isdir(module_dir):
739            shutil.rmtree(module_dir)
740        progress.progress(0.7, 'Setting up module...')
741        with closing(tarfile.open('', 'r:gz', BytesIO(tardata))) as tar:
742            tar.extractall(self.modules_dir)
743        if not os.path.isdir(module_dir):
744            raise ModuleInstallError('The archive for %s looks invalid.' % module.name)
745        # Precompile
746        compile_dir(module_dir, quiet=True)
747
748        self.versions.set(module.name, module.version)
749
750        progress.progress(0.9, 'Downloading icon...')
751        self.retrieve_icon(module)
752
753        progress.progress(1.0, 'Module %s has been installed!' % module.name)
754
755    @staticmethod
756    def url2filename(url):
757        """
758        Get a safe file name for an URL.
759
760        All non-alphanumeric characters are replaced by _.
761        """
762        return ''.join([l if l.isalnum() else '_' for l in url])
763
764    def __iter__(self):
765        for repository in self.repositories:
766            yield repository
767
768    @property
769    def errors(self):
770        errors = {}
771        for repository in self:
772            errors.update(repository.errors)
773        return errors
774
775
776class InvalidSignature(Exception):
777    def __init__(self, filename):
778        self.filename = filename
779        super(InvalidSignature, self).__init__('Invalid signature for %s' % filename)
780
781
782class Keyring(object):
783    EXTENSION = '.gpg'
784
785    def __init__(self, path):
786        self.path = path + self.EXTENSION
787        self.vpath = path + '.version'
788        self.version = 0
789
790        if self.exists():
791            with open(self.vpath, 'r') as f:
792                self.version = int(f.read().strip())
793        else:
794            if os.path.exists(self.path):
795                os.remove(self.path)
796            if os.path.exists(self.vpath):
797                os.remove(self.vpath)
798
799    def exists(self):
800        if not os.path.exists(self.vpath):
801            return False
802        if os.path.exists(self.path):
803            # Check the file is not empty.
804            # This is because there was a bug creating empty keyring files.
805            with open(self.path, 'rb') as fp:
806                if len(fp.read().strip()):
807                    return True
808        return False
809
810    def save(self, keyring_data, version):
811        with open(self.path, 'wb') as fp:
812            fp.write(keyring_data)
813        self.version = version
814        with open_for_config(self.vpath) as fp:
815            fp.write(str(version))
816
817    @staticmethod
818    def find_gpgv():
819        return find_exe('gpgv2') or find_exe('gpgv')
820
821    @staticmethod
822    def find_gpg():
823        return find_exe('gpg2') or find_exe('gpg')
824
825    def is_valid(self, data, sigdata):
826        """
827        Check if the data is signed by an accepted key.
828        data and sigdata should be strings.
829        """
830        gpg = self.find_gpg()
831        gpgv = self.find_gpgv()
832
833        if gpg:
834            from tempfile import mkdtemp
835            gpg_homedir = mkdtemp(prefix='weboob_gpg_')
836            verify_command = [gpg, '--verify', '--no-options',
837                              '--no-default-keyring', '--quiet',
838                              '--homedir', gpg_homedir]
839        elif gpgv:
840            verify_command = [gpgv]
841
842        from tempfile import NamedTemporaryFile
843        with NamedTemporaryFile(suffix='.sig', delete=False) as sigfile:
844            temp_filename = sigfile.name
845            return_code = None
846            out = ''
847            err = ''
848            try:
849                sigfile.write(sigdata)
850                sigfile.flush()  # very important
851                sigfile.close()
852                assert isinstance(data, bytes)
853                # Yes, all of it is necessary
854                proc = subprocess.Popen(verify_command + [
855                        '--status-fd', '1',
856                        '--keyring', os.path.realpath(self.path),
857                        os.path.realpath(sigfile.name),
858                        '-'],
859                    stdin=subprocess.PIPE,
860                    stdout=subprocess.PIPE,
861                    stderr=subprocess.PIPE)
862                out, err = proc.communicate(data)
863                return_code = proc.returncode
864            finally:
865                os.unlink(temp_filename)
866                if gpg:
867                    shutil.rmtree(gpg_homedir)
868
869            if return_code or b'GOODSIG' not in out or b'VALIDSIG' not in out:
870                print(out, err, file=sys.stderr)
871                return False
872        return True
873
874    def __str__(self):
875        if self.exists():
876            with open(self.path, 'rb') as f:
877                h = hashlib.sha1(f.read()).hexdigest()
878            return 'Keyring version %s, checksum %s' % (self.version, h)
879        return 'NO KEYRING'
880