Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/core/lister_transports.py
| # Copyright (C) 2017-2018 the Software Heritage developers | # Copyright (C) 2017-2018 the Software Heritage developers | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import abc | import abc | ||||
| import random | import random | ||||
| from datetime import datetime | from datetime import datetime | ||||
| from email.utils import parsedate | from email.utils import parsedate | ||||
| from pprint import pformat | from pprint import pformat | ||||
| import logging | |||||
| import requests | import requests | ||||
| import xmltodict | import xmltodict | ||||
| try: | try: | ||||
| from swh.lister._version import __version__ | from swh.lister._version import __version__ | ||||
| except ImportError: | except ImportError: | ||||
| __version__ = 'devel' | __version__ = 'devel' | ||||
| from .abstractattribute import AbstractAttribute | from .abstractattribute import AbstractAttribute | ||||
| from .lister_base import FetchError | from .lister_base import FetchError | ||||
| logger = logging.getLogger(__name__) | |||||
| class SWHListerHttpTransport(abc.ABC): | class SWHListerHttpTransport(abc.ABC): | ||||
| """Use the Requests library for making Lister endpoint requests. | """Use the Requests library for making Lister endpoint requests. | ||||
| To be used in conjunction with SWHListerBase or a subclass of it. | To be used in conjunction with SWHListerBase or a subclass of it. | ||||
| """ | """ | ||||
| PATH_TEMPLATE = AbstractAttribute('string containing a python string' | PATH_TEMPLATE = AbstractAttribute('string containing a python string' | ||||
| ' format pattern that produces the API' | ' format pattern that produces the API' | ||||
| ▲ Show 20 Lines • Show All 78 Lines • ▼ Show 20 Lines | def _transport_action(self, identifier, method='get'): | ||||
| params = self.request_params(identifier) | params = self.request_params(identifier) | ||||
| try: | try: | ||||
| if method == 'head': | if method == 'head': | ||||
| response = self.session.head(path, **params) | response = self.session.head(path, **params) | ||||
| else: | else: | ||||
| response = self.session.get(path, **params) | response = self.session.get(path, **params) | ||||
| except requests.exceptions.ConnectionError as e: | except requests.exceptions.ConnectionError as e: | ||||
| logger.warning('Failed to fetch %s: %s', path, e) | |||||
| raise FetchError(e) | raise FetchError(e) | ||||
| else: | else: | ||||
| if response.status_code not in self.EXPECTED_STATUS_CODES: | if response.status_code not in self.EXPECTED_STATUS_CODES: | ||||
| raise FetchError(response) | raise FetchError(response) | ||||
| return response | return response | ||||
| def transport_head(self, identifier): | def transport_head(self, identifier): | ||||
| """Retrieve head information on api. | """Retrieve head information on api. | ||||
| ▲ Show 20 Lines • Show All 51 Lines • Show Last 20 Lines | |||||