Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/core/lister_transports.py
# Copyright (C) 2017-2018 the Software Heritage developers | # Copyright (C) 2017-2018 the Software Heritage developers | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import abc | import abc | ||||
import random | import random | ||||
from datetime import datetime | from datetime import datetime | ||||
from email.utils import parsedate | from email.utils import parsedate | ||||
from pprint import pformat | from pprint import pformat | ||||
import logging | import logging | ||||
import requests | import requests | ||||
import xmltodict | import xmltodict | ||||
from typing import Optional, Union | |||||
try: | try: | ||||
from swh.lister._version import __version__ | from swh.lister._version import __version__ | ||||
except ImportError: | except ImportError: | ||||
__version__ = 'devel' | __version__ = 'devel' | ||||
from .abstractattribute import AbstractAttribute | from .abstractattribute import AbstractAttribute | ||||
from .lister_base import FetchError | from .lister_base import FetchError | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
class ListerHttpTransport(abc.ABC): | class ListerHttpTransport(abc.ABC): | ||||
"""Use the Requests library for making Lister endpoint requests. | """Use the Requests library for making Lister endpoint requests. | ||||
To be used in conjunction with ListerBase or a subclass of it. | To be used in conjunction with ListerBase or a subclass of it. | ||||
""" | """ | ||||
DEFAULT_URL = None | DEFAULT_URL = None # type: Optional[str] | ||||
PATH_TEMPLATE = AbstractAttribute('string containing a python string' | PATH_TEMPLATE = \ | ||||
' format pattern that produces the API' | AbstractAttribute( | ||||
' endpoint path for listing stored' | 'string containing a python string format pattern that produces' | ||||
' repositories when given an index.' | ' the API endpoint path for listing stored repositories when given' | ||||
' eg. "/repositories?after=%s".' | ' an index, e.g., "/repositories?after=%s". To be implemented in' | ||||
'To be implemented in the API-specific' | ' the API-specific class inheriting this.' | ||||
' class inheriting this.') | ) # type: Union[AbstractAttribute, Optional[str]] | ||||
EXPECTED_STATUS_CODES = (200, 429, 403, 404) | EXPECTED_STATUS_CODES = (200, 429, 403, 404) | ||||
def request_headers(self): | def request_headers(self): | ||||
"""Returns dictionary of any request headers needed by the server. | """Returns dictionary of any request headers needed by the server. | ||||
MAY BE OVERRIDDEN if request headers are needed. | MAY BE OVERRIDDEN if request headers are needed. | ||||
""" | """ | ||||
▲ Show 20 Lines • Show All 161 Lines • ▼ Show 20 Lines | |||||
class ListerOnePageApiTransport(ListerHttpTransport): | class ListerOnePageApiTransport(ListerHttpTransport): | ||||
"""Leverage requests library to retrieve basic html page and parse | """Leverage requests library to retrieve basic html page and parse | ||||
result. | result. | ||||
To be used in conjunction with ListerBase or a subclass of it. | To be used in conjunction with ListerBase or a subclass of it. | ||||
""" | """ | ||||
PAGE = AbstractAttribute("The server api's unique page to retrieve and " | PAGE = AbstractAttribute( | ||||
"parse for information") | "URL of the API's unique page to retrieve and parse " | ||||
"for information") # type: Union[AbstractAttribute, str] | |||||
PATH_TEMPLATE = None # we do not use it | PATH_TEMPLATE = None # we do not use it | ||||
def __init__(self, url=None): | def __init__(self, url=None): | ||||
self.session = requests.Session() | self.session = requests.Session() | ||||
self.lister_version = __version__ | self.lister_version = __version__ | ||||
def request_uri(self, _): | def request_uri(self, _): | ||||
"""Get the full request URI given the transport_request identifier. | """Get the full request URI given the transport_request identifier. | ||||
""" | """ | ||||
return self.PAGE | return self.PAGE |