1# -*- coding: utf-8 -*- 2 3# Copyright(C) 2010-2014 Romain Bignon 4# 5# This file is part of weboob. 6# 7# weboob is free software: you can redistribute it and/or modify 8# it under the terms of the GNU Lesser General Public License as published by 9# the Free Software Foundation, either version 3 of the License, or 10# (at your option) any later version. 11# 12# weboob is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU Lesser General Public License for more details. 16# 17# You should have received a copy of the GNU Lesser General Public License 18# along with weboob. If not, see <http://www.gnu.org/licenses/>. 19 20 21import os 22 23from weboob.core.bcall import BackendsCall 24from weboob.core.modules import ModulesLoader, RepositoryModulesLoader 25from weboob.core.backendscfg import BackendsConfig 26from weboob.core.requests import RequestsManager 27from weboob.core.repositories import Repositories, PrintProgress 28from weboob.core.scheduler import Scheduler 29from weboob.tools.backend import Module 30from weboob.tools.compat import basestring, unicode 31from weboob.tools.config.iconfig import ConfigError 32from weboob.tools.log import getLogger 33from weboob.exceptions import ModuleLoadError 34 35 36__all__ = ['WebNip', 'Weboob'] 37 38 39class VersionsMismatchError(ConfigError): 40 pass 41 42 43class WebNip(object): 44 """ 45 Weboob in Non Integrated Programs 46 47 It provides methods to build backends or call methods on all loaded 48 backends. 49 50 You should use this class when you want to build an application 51 using Weboob as a library, without using the standard modules nor 52 the automatic module download and update machanism. When using 53 WebNip, you have to explicitely provide module paths and deal 54 yourself with backend configuration. 55 56 :param modules_path: path to directory containing modules. 57 :type modules_path: :class:`basestring` 58 :param storage: provide a storage where backends can save data 59 :type storage: :class:`weboob.tools.storage.IStorage` 60 :param scheduler: what scheduler to use; default is :class:`weboob.core.scheduler.Scheduler` 61 :type scheduler: :class:`weboob.core.scheduler.IScheduler` 62 """ 63 VERSION = '2.0' 64 65 def __init__(self, modules_path=None, storage=None, scheduler=None): 66 self.logger = getLogger('weboob') 67 self.backend_instances = {} 68 self.requests = RequestsManager() 69 70 if modules_path is None: 71 import pkg_resources 72 # Package weboob_modules is provided by 73 # https://git.weboob.org/weboob/modules 74 # and should be pip-installed separately. 75 # Note that Weboob users should rather install Weboob modules 76 # through https://updates.weboob.org/. 77 modules_path = pkg_resources.resource_filename('weboob_modules', '') 78 79 if modules_path: 80 self.modules_loader = ModulesLoader(modules_path, self.VERSION) 81 82 if scheduler is None: 83 scheduler = Scheduler() 84 self.scheduler = scheduler 85 86 self.storage = storage 87 88 def __deinit__(self): 89 self.deinit() 90 91 def deinit(self): 92 """ 93 Call this method when you stop using Weboob, to 94 properly unload all correctly. 95 """ 96 self.unload_backends() 97 98 def build_backend(self, module_name, params=None, storage=None, name=None, nofail=False, logger=None): 99 """ 100 Create a backend. 101 102 It does not load it into the Weboob object, so you are responsible for 103 deinitialization and calls. 104 105 :param module_name: name of module 106 :param params: parameters to give to backend 107 :type params: :class:`dict` 108 :param storage: storage to use 109 :type storage: :class:`weboob.tools.storage.IStorage` 110 :param name: name of backend 111 :type name: :class:`basestring` 112 :rtype: :class:`weboob.tools.backend.Module` 113 :param nofail: if true, this call can't fail 114 :type nofail: :class:`bool` 115 """ 116 module = self.modules_loader.get_or_load_module(module_name) 117 118 backend_instance = module.create_instance(self, name or module_name, params or {}, storage, nofail, logger=logger or self.logger) 119 return backend_instance 120 121 class LoadError(Exception): 122 """ 123 Raised when a backend is unabled to load. 124 125 :param backend_name: name of backend we can't load 126 :param exception: exception object 127 """ 128 129 def __init__(self, backend_name, exception): 130 super(WebNip.LoadError, self).__init__(unicode(exception)) 131 self.backend_name = backend_name 132 133 def load_backend(self, module_name, name, params=None, storage=None): 134 """ 135 Load a backend. 136 137 :param module_name: name of module to load 138 :type module_name: :class:`basestring`: 139 :param name: name of instance 140 :type name: :class:`basestring` 141 :param params: parameters to give to backend 142 :type params: :class:`dict` 143 :param storage: storage to use 144 :type storage: :class:`weboob.tools.storage.IStorage` 145 :rtype: :class:`weboob.tools.backend.Module` 146 """ 147 if name is None: 148 name = module_name 149 150 if name in self.backend_instances: 151 raise self.LoadError(name, 'A loaded backend already named "%s"' % name) 152 153 backend = self.build_backend(module_name, params, storage, name) 154 self.backend_instances[name] = backend 155 return backend 156 157 def unload_backends(self, names=None): 158 """ 159 Unload backends. 160 161 :param names: if specified, only unload that backends 162 :type names: :class:`list` 163 """ 164 unloaded = {} 165 if isinstance(names, basestring): 166 names = [names] 167 elif names is None: 168 names = list(self.backend_instances.keys()) 169 170 for name in names: 171 backend = self.backend_instances.pop(name) 172 with backend: 173 backend.deinit() 174 unloaded[backend.name] = backend 175 176 return unloaded 177 178 def __getitem__(self, name): 179 """ 180 Alias for :func:`WebNip.get_backend`. 181 """ 182 return self.get_backend(name) 183 184 def get_backend(self, name, **kwargs): 185 """ 186 Get a backend from its name. 187 188 :param name: name of backend to get 189 :type name: str 190 :param default: if specified, get this value when the backend is not found 191 :type default: whatever you want 192 :raises: :class:`KeyError` if not found. 193 """ 194 try: 195 return self.backend_instances[name] 196 except KeyError: 197 if 'default' in kwargs: 198 return kwargs['default'] 199 else: 200 raise 201 202 def count_backends(self): 203 """ 204 Get number of loaded backends. 205 """ 206 return len(self.backend_instances) 207 208 def iter_backends(self, caps=None, module=None): 209 """ 210 Iter on each backends. 211 212 Note: each backend is locked when it is returned. 213 214 :param caps: optional list of capabilities to select backends 215 :type caps: tuple[:class:`weboob.capabilities.base.Capability`] 216 :param module: optional name of module 217 :type module: :class:`basestring` 218 :rtype: iter[:class:`weboob.tools.backend.Module`] 219 """ 220 for _, backend in sorted(self.backend_instances.items()): 221 if (caps is None or backend.has_caps(caps)) and \ 222 (module is None or backend.NAME == module): 223 with backend: 224 yield backend 225 226 def __getattr__(self, name): 227 def caller(*args, **kwargs): 228 return self.do(name, *args, **kwargs) 229 return caller 230 231 def do(self, function, *args, **kwargs): 232 r""" 233 Do calls on loaded backends with specified arguments, in separated 234 threads. 235 236 This function has two modes: 237 238 - If *function* is a string, it calls the method with this name on 239 each backends with the specified arguments; 240 - If *function* is a callable, it calls it in a separated thread with 241 the locked backend instance at first arguments, and \*args and 242 \*\*kwargs. 243 244 :param function: backend's method name, or a callable object 245 :type function: :class:`str` 246 :param backends: list of backends to iterate on 247 :type backends: list[:class:`str`] 248 :param caps: iterate on backends which implement this caps 249 :type caps: list[:class:`weboob.capabilities.base.Capability`] 250 :rtype: A :class:`weboob.core.bcall.BackendsCall` object (iterable) 251 """ 252 backends = list(self.backend_instances.values()) 253 _backends = kwargs.pop('backends', None) 254 if _backends is not None: 255 if isinstance(_backends, Module): 256 backends = [_backends] 257 elif isinstance(_backends, basestring): 258 if len(_backends) > 0: 259 try: 260 backends = [self.backend_instances[_backends]] 261 except (ValueError, KeyError): 262 backends = [] 263 elif isinstance(_backends, (list, tuple, set)): 264 backends = [] 265 for backend in _backends: 266 if isinstance(backend, basestring): 267 try: 268 backends.append(self.backend_instances[backend]) 269 except (ValueError, KeyError): 270 pass 271 else: 272 backends.append(backend) 273 else: 274 self.logger.warning(u'The "backends" value isn\'t supported: %r', _backends) 275 276 if 'caps' in kwargs: 277 caps = kwargs.pop('caps') 278 backends = [backend for backend in backends if backend.has_caps(caps)] 279 280 # The return value MUST BE the BackendsCall instance. Please never iterate 281 # here on this object, because caller might want to use other methods, like 282 # wait() on callback_thread(). 283 # Thanks a lot. 284 return BackendsCall(backends, function, *args, **kwargs) 285 286 def schedule(self, interval, function, *args): 287 """ 288 Schedule an event. 289 290 :param interval: delay before calling the function 291 :type interval: int 292 :param function: function to call 293 :type function: callabale 294 :param args: arguments to give to function 295 :returns: an event identificator 296 """ 297 return self.scheduler.schedule(interval, function, *args) 298 299 def repeat(self, interval, function, *args): 300 """ 301 Repeat a call to a function 302 303 :param interval: interval between two calls 304 :type interval: int 305 :param function: function to call 306 :type function: callable 307 :param args: arguments to give to function 308 :returns: an event identificator 309 """ 310 return self.scheduler.repeat(interval, function, *args) 311 312 def cancel(self, ev): 313 """ 314 Cancel an event 315 316 :param ev: the event identificator 317 """ 318 return self.scheduler.cancel(ev) 319 320 def want_stop(self): 321 """ 322 Plan to stop the scheduler. 323 """ 324 return self.scheduler.want_stop() 325 326 def loop(self): 327 """ 328 Run the scheduler loop 329 """ 330 return self.scheduler.run() 331 332 def load_or_install_module(self, module_name): 333 """ Load a backend, but can't install it """ 334 return self.modules_loader.get_or_load_module(module_name) 335 336 337class Weboob(WebNip): 338 """ 339 The main class of Weboob, used to manage backends, modules repositories and 340 call methods on all loaded backends. 341 342 :param workdir: optional parameter to set path of the working directory 343 :type workdir: str 344 :param datadir: optional parameter to set path of the data directory 345 :type datadir: str 346 :param backends_filename: name of the *backends* file, where configuration of 347 backends is stored 348 :type backends_filename: str 349 :param storage: provide a storage where backends can save data 350 :type storage: :class:`weboob.tools.storage.IStorage` 351 """ 352 BACKENDS_FILENAME = 'backends' 353 354 def __init__(self, workdir=None, datadir=None, backends_filename=None, scheduler=None, storage=None): 355 super(Weboob, self).__init__(modules_path=False, scheduler=scheduler, storage=storage) 356 357 # Create WORKDIR 358 if workdir is None: 359 if 'WEBOOB_WORKDIR' in os.environ: 360 workdir = os.environ['WEBOOB_WORKDIR'] 361 else: 362 workdir = os.path.join(os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')), 'weboob') 363 364 self.workdir = os.path.realpath(workdir) 365 self._create_dir(workdir) 366 367 # Create DATADIR 368 if datadir is None: 369 if 'WEBOOB_DATADIR' in os.environ: 370 datadir = os.environ['WEBOOB_DATADIR'] 371 elif 'WEBOOB_WORKDIR' in os.environ: 372 datadir = os.environ['WEBOOB_WORKDIR'] 373 else: 374 datadir = os.path.join(os.environ.get('XDG_DATA_HOME', os.path.join(os.path.expanduser('~'), '.local', 'share')), 'weboob') 375 376 _datadir = os.path.realpath(datadir) 377 self._create_dir(_datadir) 378 379 # Modules management 380 self.repositories = Repositories(workdir, _datadir, self.VERSION) 381 self.modules_loader = RepositoryModulesLoader(self.repositories) 382 383 # Backend instances config 384 if not backends_filename: 385 backends_filename = os.environ.get('WEBOOB_BACKENDS', os.path.join(self.workdir, self.BACKENDS_FILENAME)) 386 elif not backends_filename.startswith('/'): 387 backends_filename = os.path.join(self.workdir, backends_filename) 388 self.backends_config = BackendsConfig(backends_filename) 389 390 def _create_dir(self, name): 391 if not os.path.exists(name): 392 os.makedirs(name) 393 elif not os.path.isdir(name): 394 self.logger.error(u'"%s" is not a directory', name) 395 396 def update(self, progress=PrintProgress()): 397 """ 398 Update modules from repositories. 399 """ 400 self.repositories.update(progress) 401 402 modules_to_check = set([module_name for _, module_name, _ in self.backends_config.iter_backends()]) 403 for module_name in modules_to_check: 404 minfo = self.repositories.get_module_info(module_name) 405 if minfo and not minfo.is_installed(): 406 self.repositories.install(minfo, progress) 407 408 def build_backend(self, module_name, params=None, storage=None, name=None, nofail=False): 409 """ 410 Create a single backend which is not listed in configuration. 411 412 :param module_name: name of module 413 :param params: parameters to give to backend 414 :type params: :class:`dict` 415 :param storage: storage to use 416 :type storage: :class:`weboob.tools.storage.IStorage` 417 :param name: name of backend 418 :type name: :class:`basestring` 419 :rtype: :class:`weboob.tools.backend.Module` 420 :param nofail: if true, this call can't fail 421 :type nofail: :class:`bool` 422 """ 423 minfo = self.repositories.get_module_info(module_name) 424 if minfo is None: 425 raise ModuleLoadError(module_name, 'Module does not exist.') 426 427 if not minfo.is_installed(): 428 self.repositories.install(minfo) 429 430 return super(Weboob, self).build_backend(module_name, params, storage, name, nofail) 431 432 def load_backends(self, caps=None, names=None, modules=None, exclude=None, storage=None, errors=None): 433 """ 434 Load backends listed in config file. 435 436 :param caps: load backends which implement all of specified caps 437 :type caps: tuple[:class:`weboob.capabilities.base.Capability`] 438 :param names: load backends in list 439 :type names: tuple[:class:`str`] 440 :param modules: load backends which module is in list 441 :type modules: tuple[:class:`str`] 442 :param exclude: do not load backends in list 443 :type exclude: tuple[:class:`str`] 444 :param storage: use this storage if specified 445 :type storage: :class:`weboob.tools.storage.IStorage` 446 :param errors: if specified, store every errors in this list 447 :type errors: list[:class:`LoadError`] 448 :returns: loaded backends 449 :rtype: dict[:class:`str`, :class:`weboob.tools.backend.Module`] 450 """ 451 loaded = {} 452 if storage is None: 453 storage = self.storage 454 455 if not self.repositories.check_repositories(): 456 self.logger.error(u'Repositories are not consistent with the sources.list') 457 raise VersionsMismatchError(u'Versions mismatch, please run "weboob-config update"') 458 459 for backend_name, module_name, params in self.backends_config.iter_backends(): 460 if '_enabled' in params and not params['_enabled'].lower() in ('1', 'y', 'true', 'on', 'yes') or \ 461 names is not None and backend_name not in names or \ 462 modules is not None and module_name not in modules or \ 463 exclude is not None and backend_name in exclude: 464 continue 465 466 minfo = self.repositories.get_module_info(module_name) 467 if minfo is None: 468 self.logger.warning(u'Backend "%s" is referenced in %s but was not found. ' 469 u'Perhaps a missing repository or a removed module?', module_name, self.backends_config.confpath) 470 continue 471 472 if caps is not None and not minfo.has_caps(caps): 473 continue 474 475 if not minfo.is_installed(): 476 self.repositories.install(minfo) 477 478 module = None 479 try: 480 module = self.modules_loader.get_or_load_module(module_name) 481 except ModuleLoadError as e: 482 self.logger.error(u'Unable to load module "%s": %s', module_name, e) 483 continue 484 485 if backend_name in self.backend_instances: 486 self.logger.warning(u'Oops, the backend "%s" is already loaded. Unload it before reloading...', backend_name) 487 self.unload_backends(backend_name) 488 489 try: 490 backend_instance = module.create_instance(self, backend_name, params, storage) 491 except Module.ConfigError as e: 492 if errors is not None: 493 errors.append(self.LoadError(backend_name, e)) 494 else: 495 self.backend_instances[backend_name] = loaded[backend_name] = backend_instance 496 return loaded 497 498 def load_or_install_module(self, module_name): 499 """ Load a backend, and install it if not done before """ 500 try: 501 return self.modules_loader.get_or_load_module(module_name) 502 except ModuleLoadError: 503 self.repositories.install(module_name) 504 return self.modules_loader.get_or_load_module(module_name) 505