Source code for

# Copyright(C) 2010-2023 Romain Bignon
# This file is part of woob.
# woob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# woob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# GNU Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public License
# along with woob. If not, see <>.

from __future__ import annotations

import logging
import importlib
import os
from copy import copy
from threading import RLock
import warnings
from urllib.request import getproxies
from typing import List, Any, Dict, Tuple, Iterator, Type, ClassVar, TYPE_CHECKING

from packaging.version import Version

from woob import __version__
from woob.capabilities.base import BaseObject, Capability, FieldNotFound, NotAvailable, NotLoaded
from import json
from import getLogger
from import iter_fields
from import IStorage
from import ValuesDict, ValueBool

    from woob.core import WoobBase
    from woob.browser import Browser

__all__ = ['BackendStorage', 'BackendConfig', 'Module']

[docs]class BackendStorage: """ This is an abstract layer to store data in storages (:mod:``) easily. It is instancied automatically in constructor of :class:`Module`, in the :attr:`` attribute. :param name: name of backend :type name: str :param storage: storage object :type storage: :class:`` """ def __init__(self, name: str, storage: IStorage | None): = name = storage
[docs] def set(self, *args): """ Set value in the storage. Example: >>> from import StandardStorage >>> backend = BackendStorage('blah', StandardStorage('/tmp/cfg')) >>>'config', 'nb_of_threads', 10) # doctest: +SKIP >>> :param args: the path where to store value """ if'backends',, *args)
[docs] def delete(self, *args): """ Delete a value from the storage. :param args: path to delete. """ if'backends',, *args)
[docs] def get(self, *args, **kwargs) -> Any: """ Get a value or a dict of values in storage. Example: >>> from import StandardStorage >>> backend = BackendStorage('blah', StandardStorage('/tmp/cfg')) >>>'config', 'nb_of_threads') # doctest: +SKIP 10 >>>'config', 'unexistant', 'path', default='lol') # doctest: +SKIP 'lol' >>>'config') # doctest: +SKIP {'nb_of_threads': 10, 'other_things': 'blah'} :param args: path to get :param default: if specified, default value when path is not found """ if return'backends',, *args, **kwargs) return kwargs.get('default', None)
[docs] def load(self, default: Dict): """ Load storage. It is made automatically when your backend is created, and use the ``STORAGE`` class attribute as default. :param default: this is the default tree if storage is empty :type default: :class:`dict` """ if'backends',, default)
[docs] def save(self): """ Save storage. """ if'backends',
[docs]class BackendConfig(ValuesDict): """ Configuration of a backend. This class is firstly instanced as a :class:``, containing some :class:`` (and derivated) objects. Then, using the :func:`load` method will load configuration from file and create a copy of the :class:`BackendConfig` object with the loaded values. """ modname: str instname: str woob: WoobBase
[docs] def load( self, woob: WoobBase, modname: str, instname: str, config: Dict, nofail: bool = False ) -> BackendConfig: """ Load configuration from dict to create an instance. :param woob: woob object :type woob: :class:`woob.core.woob.WoobBase` :param modname: name of the module :type modname: :class:`str` :param instname: name of this backend :type instname: :class:`str` :param params: parameters to load :type params: :class:`dict` :param nofail: if true, this call can't fail :type nofail: :class:`bool` :rtype: :class:`BackendConfig` """ cfg = self.__class__() cfg.modname = modname cfg.instname = instname cfg.woob = woob for name, field in self.items(): value = config.get(name, None) if value is None: if not nofail and field.required: raise Module.ConfigError( f'Backend({cfg.instname}): Configuration error: Missing parameter {name} ({field.description})', bad_fields=[name] ) value = field.default field = copy(field) try: field.load(cfg.instname, value, cfg.woob.requests) except ValueError as v: if not nofail: raise Module.ConfigError( f'Backend({cfg.instname}): Configuration error for field "{name}": {v}', bad_fields=[name] ) cfg[name] = field return cfg
[docs] def dump(self) -> dict: """ Dump config in a dictionary. :rtype: :class:`dict` """ settings = {} for name, value in self.items(): if not value.transient: settings[name] = value.dump() return settings
[docs] def save(self, edit: bool = True, params: Dict | None = None): """ Save backend config. :param edit: if true, it changes config of an existing backend :type edit: :class:`bool` :param params: if supplied, params to merge with the ones of the current object :type params: :class:`dict` """ assert self.modname is not None assert self.instname is not None assert self.woob is not None dump = self.dump() if params is not None: dump.update(params) if edit: self.woob.backends_config.edit_backend(self.instname, dump) else: self.woob.backends_config.add_backend(self.instname, self.modname, dump)
[docs]class Module: """ Base class for modules. You may derivate it, and also all capabilities you want to implement. :param woob: woob instance :type woob: :class:`woob.core.woob.Woob` :param name: name of backend :type name: :class:`str` :param config: configuration of backend (optional) :type config: :class:`dict` :param storage: storage object (optional) :type storage: :class:`` :param logger: parent logger (optional) :type logger: :class:`logging.Logger` """ NAME: ClassVar[str] """Name of the maintainer of this module.""" MAINTAINER: ClassVar[str] = '<unspecified>' """Name of the maintainer.""" EMAIL: ClassVar[str] = '<unspecified>' """Email address of the maintainer.""" DESCRIPTION: ClassVar[str] = '<unspecified>' """Description""" LICENSE: ClassVar[str] = '<unspecified>' """License of the module""" CONFIG: ClassVar[BackendConfig] = BackendConfig() """Configuration required for backends. Values must be :class:`` objects. """ STORAGE: ClassVar[Dict] = {} """Storage""" BROWSER: Browser | None = None """Browser class""" ICON: ClassVar[str | None] = None """URL to an optional icon. If you want to create your own icon, create a 'favicon.png' icon in the module's directory, and keep the ICON value to None. """ OBJECTS: ClassVar[Dict] = {} """Supported objects to fill The key is the class and the value the method to call to fill Method prototype: method(object, fields) When the method is called, fields are only the one which are NOT yet filled. """ DEPENDENCIES: ClassVar[Tuple[str]] = () """Tuple of module names on which this module depends."""
[docs] class ConfigError(Exception): """ Raised when the config can't be loaded. """ def __init__(self, message, bad_fields=None): """ :type message: str :param message: message of the exception :type bad_fields: list[str] :param bad_fields: names of the config fields which are incorrect """ super().__init__(message) self.bad_fields = bad_fields or ()
def __enter__(self): self.lock.acquire() def __exit__(self, t, v, tb): self.lock.release() def __repr__(self): return f"<Backend {}>" def __new__(cls, *args, **kwargs): """ Accept any arguments, necessary for AbstractModule __new__ override. AbstractModule, in its overridden __new__, removes itself from class hierarchy so its __new__ is called only once. In python 3, default (object) __new__ is then used for next instantiations but it's a slot/"fixed" version supporting only one argument (type to instanciate). """ return object.__new__(cls) @property def VERSION(self): warnings.warn( 'Attribute Module.VERSION will be removed in woob 4, do not use it.', DeprecationWarning, stacklevel=3 ) return Version(__version__).base_version def __init__( self, woob: WoobBase, name: str, config: Dict | None = None, storage: IStorage | None = None, logger: logging.Logger | None = None, nofail: bool = False ): if ( hasattr(self.__class__, 'VERSION') and not isinstance(self.__class__.VERSION, property) ): warnings.warn( f'Class attribute {self.__class__.__name__}.VERSION is now ' 'unused and deprecated, you can remove it. ' 'If you do so, do not forget to increase the woob version to at ' 'least 3.4 in requirements.txt.', DeprecationWarning, ) self.logger = getLogger(name, parent=logger) self.woob = woob = name self.lock = RLock() if config is None: config = {} # Private fields (which start with '_') self._private_config = dict((key, value) for key, value in config.items() if key.startswith('_')) # Load configuration of backend. self.config = self.CONFIG.load(woob, self.NAME,, config, nofail) = BackendStorage(, storage)
[docs] def dump_state(self): """ Dump module state into storage. """ if hasattr(self.browser, 'dump_state'):'browser_state', self.browser.dump_state())
[docs] def deinit(self): """ This abstract method is called when the backend is unloaded. """ if self._browser is None: return try: self.dump_state() finally: if hasattr(self.browser, 'deinit'): self.browser.deinit()
@property def weboob(self): """ .. deprecated:: 3.4 Don't use this attribute, but :attr:`woob` instead. """ warnings.warn('Use Module.woob instead.', DeprecationWarning, stacklevel=2) return self.woob _browser = None @property def browser(self) -> Browser: """ Attribute 'browser'. The browser is created at the first call of this attribute, to avoid useless pages access. Note that the :func:`create_default_browser` method is called to create it. """ if self._browser is None: self._browser = self.create_default_browser() return self._browser
[docs] def create_default_browser(self) -> Browser | None: """ Method to overload to build the default browser in attribute 'browser'. """ return self.create_browser()
[docs] def create_browser(self, *args, **kwargs) -> Browser | None: """ Build a browser from the BROWSER class attribute and the given arguments. :param klass: optional parameter to give another browser class to instanciate :type klass: :class:`woob.browser.browsers.Browser` """ klass = kwargs.pop('klass', self.BROWSER) if not klass: return None kwargs['proxy'] = self.get_proxy() if '_proxy_headers' in self._private_config: kwargs['proxy_headers'] = self._private_config['_proxy_headers'] if isinstance(kwargs['proxy_headers'], str): kwargs['proxy_headers'] = json.loads(kwargs['proxy_headers']) if '_ssl_verify' in self._private_config: # value can be either a boolean or a string (path) value = ValueBool() try: value.set(self._private_config['_ssl_verify']) except ValueError: kwargs.setdefault('verify', self._private_config['_ssl_verify']) else: kwargs.setdefault('verify', value.get()) kwargs['logger'] = self.logger if self.logger.settings['responses_dirname']: kwargs.setdefault('responses_dirname', os.path.join(self.logger.settings['responses_dirname'], self._private_config.get('_debug_dir', elif os.path.isabs(self._private_config.get('_debug_dir', '')): kwargs.setdefault('responses_dirname', self._private_config['_debug_dir']) if '_highlight_el' in self._private_config: value = ValueBool() try: value.set(self._private_config['_highlight_el']) except ValueError as e: raise Module.ConfigError( f'Backend({}): Configuration error: _highlight_el must be a boolean' ) from e kwargs.setdefault('highlight_el', value.get()) browser = klass(*args, **kwargs) if hasattr(browser, 'load_state'): browser.load_state('browser_state', default={})) return browser
[docs] def get_proxy(self) -> Dict[str, str]: """ Get proxy to use. It will read in environment variables, then in backend config. Proxy keys in backend config are: * ``_proxy`` for HTTP requests * ``_proxy_ssl`` for HTTPS requests """ # Get proxies from environment variables proxies = getproxies() # Override them with backend-specific config if '_proxy' in self._private_config: proxies['http'] = self._private_config['_proxy'] if '_proxy_ssl' in self._private_config: proxies['https'] = self._private_config['_proxy_ssl'] # Remove empty values for key in list(proxies.keys()): if not proxies[key]: del proxies[key] return proxies
[docs] @classmethod def iter_caps(cls) -> Iterator[Type[Capability]]: """ Iter capabilities implemented by this backend. :rtype: iter[:class:`woob.capabilities.base.Capability`] """ for base in cls.mro(): if issubclass(base, Capability) and base != Capability and base != cls and not issubclass(base, Module): yield base
[docs] def has_caps(self, *caps) -> bool: """ Check if this backend implements at least one of these capabilities. `caps` should be list of :class:`Capability` objects (e.g. :class:`CapBank`) or capability names (e.g. 'bank'). """ available_cap_names = [cap.__name__ for cap in self.iter_caps()] return any( (isinstance(c, str) and c in available_cap_names) or isinstance(self, c) for c in caps )
[docs] def fillobj(self, obj: BaseObject, fields: List[str] | None = None): """ Fill an object with the wanted fields. :param fields: what fields to fill; if None, all fields are filled :type fields: :class:`list` """ if obj is None: return obj def not_loaded_or_incomplete(v): return (v is NotLoaded or isinstance(v, BaseObject) and not v.__iscomplete__()) def not_loaded(v): return v is NotLoaded def filter_missing_fields(obj, fields, check_cb): missing_fields = [] if fields is None: # Select all fields if isinstance(obj, BaseObject): fields = [item[0] for item in obj.iter_fields()] else: fields = [item[0] for item in iter_fields(obj)] for field in fields: if not hasattr(obj, field): raise FieldNotFound(obj, field) value = getattr(obj, field) missing = False if hasattr(value, '__iter__'): for v in (value.values() if isinstance(value, dict) else value): if check_cb(v): missing = True break elif check_cb(value): missing = True if missing: missing_fields.append(field) return missing_fields if isinstance(fields, str): fields = (fields,) missing_fields = filter_missing_fields(obj, fields, not_loaded_or_incomplete) if not missing_fields: return obj for key, value in self.OBJECTS.items(): if isinstance(obj, key): self.logger.debug('Fill %r with fields: %s', obj, missing_fields) obj = value(self, obj, missing_fields) or obj break missing_fields = filter_missing_fields(obj, fields, not_loaded) # Object is not supported by backend. Do not notice it to avoid flooding user. # That's not so bad. for field in missing_fields: setattr(obj, field, NotAvailable) return obj
class AbstractModuleMissingParentError(Exception): pass class MetaModule(type): # we can remove this class as soon as we get rid of Abstract* def __new__(mcs, name, bases, dct): if name != 'AbstractModule' and AbstractModule in bases: warnings.warn('AbstractModule is deprecated and will be removed in woob 4.0. ' 'Use standard "from woob_modules.other_module import Module" instead.', DeprecationWarning, stacklevel=2) module = importlib.import_module('woob_modules.%s' % dct['PARENT']) symbols = [getattr(module, attr) for attr in dir(module) if not attr.startswith('__')] klass = next( symbol for symbol in symbols if isinstance(symbol, type) and issubclass(symbol, Module) and symbol != Module ) bases = tuple(klass if isinstance(base, mcs) else base for base in bases) additional_config = dct.pop('ADDITIONAL_CONFIG', None) if additional_config: dct['CONFIG'] = BackendConfig(*(list(klass.CONFIG.values()) + list(additional_config.values()))) return super().__new__(mcs, name, bases, dct) class AbstractModule(metaclass=MetaModule): """ .. deprecated:: 3.4 Don't use this class, import woob_modules.other_module.etc instead """