Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/core/lister_base.py
Show First 20 Lines • Show All 64 Lines • ▼ Show 20 Lines | """Lister core base class. | ||||
def is_within_bounds | def is_within_bounds | ||||
""" | """ | ||||
MODEL = AbstractAttribute('Subclass type (not instance)' | MODEL = AbstractAttribute('Subclass type (not instance)' | ||||
' of swh.lister.core.models.ModelBase' | ' of swh.lister.core.models.ModelBase' | ||||
' customized for a specific service.') | ' customized for a specific service.') | ||||
LISTER_NAME = AbstractAttribute("Lister's name") | LISTER_NAME = AbstractAttribute("Lister's name") | ||||
@abc.abstractmethod | |||||
def transport_request(self, identifier): | def transport_request(self, identifier): | ||||
"""Given a target endpoint identifier to query, try once to request it. | """Given a target endpoint identifier to query, try once to request it. | ||||
Implementation of this method determines the network request protocol. | Implementation of this method determines the network request protocol. | ||||
Args: | Args: | ||||
identifier (string): unique identifier for an endpoint query. | identifier (string): unique identifier for an endpoint query. | ||||
e.g. If the service indexes lists of repositories by date and | e.g. If the service indexes lists of repositories by date and | ||||
time of creation, this might be that as a formatted string. Or | time of creation, this might be that as a formatted string. Or | ||||
it might be an integer UID. Or it might be nothing. | it might be an integer UID. Or it might be nothing. | ||||
It depends on what the service needs. | It depends on what the service needs. | ||||
Returns: | Returns: | ||||
the entire request response | the entire request response | ||||
Raises: | Raises: | ||||
Will catch internal transport-dependent connection exceptions and | Will catch internal transport-dependent connection exceptions and | ||||
raise swh.lister.core.lister_base.FetchError instead. Other | raise swh.lister.core.lister_base.FetchError instead. Other | ||||
non-connection exceptions should propagate unchanged. | non-connection exceptions should propagate unchanged. | ||||
""" | """ | ||||
pass | pass | ||||
@abc.abstractmethod | |||||
def transport_response_to_string(self, response): | def transport_response_to_string(self, response): | ||||
"""Convert the server response into a formatted string for logging. | """Convert the server response into a formatted string for logging. | ||||
Implementation of this method depends on the shape of the network | Implementation of this method depends on the shape of the network | ||||
response object returned by the transport_request method. | response object returned by the transport_request method. | ||||
Args: | Args: | ||||
response: the server response | response: the server response | ||||
Returns: | Returns: | ||||
a pretty string of the response | a pretty string of the response | ||||
""" | """ | ||||
pass | pass | ||||
@abc.abstractmethod | |||||
def transport_response_simplified(self, response): | def transport_response_simplified(self, response): | ||||
"""Convert the server response into list of a dict for each repo in the | """Convert the server response into list of a dict for each repo in the | ||||
response, mapping columns in the lister's MODEL class to repo data. | response, mapping columns in the lister's MODEL class to repo data. | ||||
Implementation of this method depends on the server API spec and the | Implementation of this method depends on the server API spec and the | ||||
shape of the network response object returned by the transport_request | shape of the network response object returned by the transport_request | ||||
method. | method. | ||||
Args: | Args: | ||||
response: response object from the server. | response: response object from the server. | ||||
Returns: | Returns: | ||||
list of repo MODEL dicts | list of repo MODEL dicts | ||||
( eg. [{'uid': r['id'], etc.} for r in response.json()] ) | ( eg. [{'uid': r['id'], etc.} for r in response.json()] ) | ||||
""" | """ | ||||
pass | pass | ||||
@abc.abstractmethod | |||||
def transport_quota_check(self, response): | def transport_quota_check(self, response): | ||||
"""Check server response to see if we're hitting request rate limits. | """Check server response to see if we're hitting request rate limits. | ||||
Implementation of this method depends on the server communication | Implementation of this method depends on the server communication | ||||
protocol and API spec and the shape of the network response object | protocol and API spec and the shape of the network response object | ||||
returned by the transport_request method. | returned by the transport_request method. | ||||
Args: | Args: | ||||
▲ Show 20 Lines • Show All 416 Lines • Show Last 20 Lines |