# Copyright(C) 2014 Simon Murail
#
# This file is part of woob.
#
# woob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# woob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with woob. If not, see <http://www.gnu.org/licenses/>.
try:
from concurrent.futures import ThreadPoolExecutor
except ImportError:
ThreadPoolExecutor = None
from http import cookiejar
from requests import Session
from requests.adapters import DEFAULT_POOLSIZE
from requests.cookies import RequestsCookieJar, cookiejar_from_dict
from requests.models import PreparedRequest
from requests.sessions import merge_setting
from requests.structures import CaseInsensitiveDict
from requests.utils import get_netrc_auth
from .adapters import HTTPAdapter
[docs]def merge_hooks(request_hooks, session_hooks):
"""
Properly merges both requests and session hooks.
This is necessary because when request_hooks == {'response': []}, the
merge breaks Session hooks entirely.
Backport from request so we can use it in wheezy
"""
if session_hooks is None or session_hooks.get('response') == []:
return request_hooks
if request_hooks is None or request_hooks.get('response') == []:
return session_hooks
ret = {}
for (k, v) in request_hooks.items():
if v is not None:
ret[k] = set(v).union(session_hooks.get(k, []))
return ret
[docs]class WoobSession(Session):
[docs] def prepare_request(self, request):
"""Constructs a :class:`PreparedRequest <PreparedRequest>` for
transmission and returns it. The :class:`PreparedRequest` has settings
merged from the :class:`Request <Request>` instance and those of the
:class:`Session`.
:param request: :class:`Request` instance to prepare with this
session's settings.
"""
cookies = request.cookies or {}
# Bootstrap CookieJar.
if not isinstance(cookies, cookiejar.CookieJar):
cookies = cookiejar_from_dict(cookies)
# Merge with session cookies
merged_cookies = RequestsCookieJar()
merged_cookies.update(self.cookies)
merged_cookies.update(cookies)
# Set environment's basic authentication if not explicitly set.
auth = request.auth
if self.trust_env and not auth and not self.auth:
auth = get_netrc_auth(request.url)
p = PreparedRequest()
p.prepare(
method=request.method.upper(),
url=request.url,
files=request.files,
data=request.data,
json=request.json,
headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict),
params=merge_setting(request.params, self.params),
auth=merge_setting(auth, self.auth),
cookies=merged_cookies,
hooks=merge_hooks(request.hooks, self.hooks),
)
return p
WeboobSession = WoobSession
[docs]class FuturesSession(WoobSession):
def __init__(self, executor=None, max_workers=2, max_retries=2, adapter_class=HTTPAdapter, *args, **kwargs):
"""Creates a FuturesSession
Notes
~~~~~
* ProcessPoolExecutor is not supported b/c Response objects are
not picklable.
* If you provide both `executor` and `max_workers`, the latter is
ignored and provided executor is used as is.
"""
super(FuturesSession, self).__init__(*args, **kwargs)
if executor is None and ThreadPoolExecutor is not None:
executor = ThreadPoolExecutor(max_workers=max_workers)
# set connection pool size equal to max_workers if needed
if max_workers > DEFAULT_POOLSIZE:
adapter_kwargs = dict(pool_connections=max_workers,
pool_maxsize=max_workers,
max_retries=max_retries)
self.mount('https://', adapter_class(**adapter_kwargs))
self.mount('http://', adapter_class(**adapter_kwargs))
self.executor = executor
[docs] def send(self, *args, **kwargs):
"""Maintains the existing api for :meth:`Session.send`
Used by :meth:`request` and thus all of the higher level methods
If the `is_async` param is True, the request is processed in a
thread. Otherwise, the request is processed as usual, in a blocking way.
In all cases, it will call the `callback` parameter and return its
result when the request has been processed.
"""
if 'async' in kwargs:
import warnings
warnings.warn('Please use is_async instead of async.', DeprecationWarning)
kwargs['is_async'] = kwargs['async']
del kwargs['async']
sup = super(FuturesSession, self).send
callback = kwargs.pop('callback', lambda future, response: response)
is_async = kwargs.pop('is_async', False)
def func(*args, **kwargs):
resp = sup(*args, **kwargs)
return callback(self, resp)
if is_async:
if not self.executor:
raise ImportError('Please install python3-concurrent.futures')
return self.executor.submit(func, *args, **kwargs)
return func(*args, **kwargs)
[docs] def close(self):
super(FuturesSession, self).close()
if self.executor:
self.executor.shutdown()
