diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py --- a/swh/lister/core/lister_base.py +++ b/swh/lister/core/lister_base.py @@ -134,9 +134,9 @@ pass def filter_before_inject(self, models_list): - """Function run after transport_response_simplified but before injection - into the local db and creation of workers. Can be used to eliminate - some of the results if necessary. + """Function run after transport_response_simplified but before + injection into the local db and creation of workers. Can be + used to eliminate some of the results if necessary. MAY BE OVERRIDDEN if an intermediate Lister class needs to filter results before injection without requiring every child class to do so. diff --git a/swh/lister/core/lister_transports.py b/swh/lister/core/lister_transports.py --- a/swh/lister/core/lister_transports.py +++ b/swh/lister/core/lister_transports.py @@ -55,11 +55,12 @@ return self.api_baseurl + path def request_params(self, identifier): - """Get the full parameters passed to requests given the transport_request - identifier. + """Get the full parameters passed to requests given the + transport_request identifier. MAY BE OVERRIDDEN if something more complex than the request headers - ois needed. + is needed. + """ params = {} params['headers'] = self.request_headers() or {} @@ -70,12 +71,13 @@ return params def transport_quota_check(self, response): - """Implements SWHListerBase.transport_quota_check with standard 429 code - check for HTTP with Requests library. + """Implements SWHListerBase.transport_quota_check with standard 429 + code check for HTTP with Requests library. MAY BE OVERRIDDEN if the server notifies about rate limits in a non-standard way that doesn't use HTTP 429 and the Retry-After response header. ( https://tools.ietf.org/html/rfc6585#section-4 ) + """ if response.status_code == 429: # HTTP too many requests retry_after = response.headers.get('Retry-After', self.back_off())