Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/core/lister_base.py
Show First 20 Lines • Show All 194 Lines • ▼ Show 20 Lines | def is_within_bounds(self, inner, lower=None, upper=None): | ||||
ret = inner <= upper | ret = inner <= upper | ||||
elif upper is None: | elif upper is None: | ||||
ret = inner >= lower | ret = inner >= lower | ||||
else: | else: | ||||
ret = lower <= inner <= upper | ret = lower <= inner <= upper | ||||
self.string_pattern_check(inner, lower, upper) | self.string_pattern_check(inner, lower, upper) | ||||
except Exception as e: | except Exception as e: | ||||
logging.error(str(e) + ': %s, %s, %s' % | logger.error(str(e) + ': %s, %s, %s' % | ||||
(('inner=%s%s' % (type(inner), inner)), | (('inner=%s%s' % (type(inner), inner)), | ||||
('lower=%s%s' % (type(lower), lower)), | ('lower=%s%s' % (type(lower), lower)), | ||||
('upper=%s%s' % (type(upper), upper))) | ('upper=%s%s' % (type(upper), upper))) | ||||
) | ) | ||||
raise | raise | ||||
return ret | return ret | ||||
# You probably don't need to override anything below this line. | # You probably don't need to override anything below this line. | ||||
DEFAULT_CONFIG = { | DEFAULT_CONFIG = { | ||||
'storage': ('dict', { | 'storage': ('dict', { | ||||
Show All 28 Lines | def ADDITIONAL_CONFIG(self): # noqa: N802 | ||||
} | } | ||||
INITIAL_BACKOFF = 10 | INITIAL_BACKOFF = 10 | ||||
MAX_RETRIES = 7 | MAX_RETRIES = 7 | ||||
CONN_SLEEP = 10 | CONN_SLEEP = 10 | ||||
def __init__(self, override_config=None): | def __init__(self, override_config=None): | ||||
self.backoff = self.INITIAL_BACKOFF | self.backoff = self.INITIAL_BACKOFF | ||||
logging.debug('Loading config from %s' % self.CONFIG_BASE_FILENAME) | logger.debug('Loading config from %s' % self.CONFIG_BASE_FILENAME) | ||||
self.config = self.parse_config_file( | self.config = self.parse_config_file( | ||||
base_filename=self.CONFIG_BASE_FILENAME, | base_filename=self.CONFIG_BASE_FILENAME, | ||||
additional_configs=[self.ADDITIONAL_CONFIG] | additional_configs=[self.ADDITIONAL_CONFIG] | ||||
) | ) | ||||
self.config['cache_dir'] = os.path.expanduser(self.config['cache_dir']) | self.config['cache_dir'] = os.path.expanduser(self.config['cache_dir']) | ||||
if self.config['cache_responses']: | if self.config['cache_responses']: | ||||
config.prepare_folders(self.config, 'cache_dir') | config.prepare_folders(self.config, 'cache_dir') | ||||
Show All 31 Lines | def safely_issue_request(self, identifier): | ||||
retries_left = self.MAX_RETRIES | retries_left = self.MAX_RETRIES | ||||
do_cache = self.config['cache_responses'] | do_cache = self.config['cache_responses'] | ||||
r = None | r = None | ||||
while retries_left > 0: | while retries_left > 0: | ||||
try: | try: | ||||
r = self.transport_request(identifier) | r = self.transport_request(identifier) | ||||
except FetchError: | except FetchError: | ||||
# network-level connection error, try again | # network-level connection error, try again | ||||
logging.warning( | logger.warning( | ||||
'connection error on %s: sleep for %d seconds' % | 'connection error on %s: sleep for %d seconds' % | ||||
(identifier, self.CONN_SLEEP)) | (identifier, self.CONN_SLEEP)) | ||||
time.sleep(self.CONN_SLEEP) | time.sleep(self.CONN_SLEEP) | ||||
retries_left -= 1 | retries_left -= 1 | ||||
continue | continue | ||||
if do_cache: | if do_cache: | ||||
self.save_response(r) | self.save_response(r) | ||||
# detect throttling | # detect throttling | ||||
must_retry, delay = self.transport_quota_check(r) | must_retry, delay = self.transport_quota_check(r) | ||||
if must_retry: | if must_retry: | ||||
logging.warning( | logger.warning( | ||||
'rate limited on %s: sleep for %f seconds' % | 'rate limited on %s: sleep for %f seconds' % | ||||
(identifier, delay)) | (identifier, delay)) | ||||
time.sleep(delay) | time.sleep(delay) | ||||
else: # request ok | else: # request ok | ||||
break | break | ||||
retries_left -= 1 | retries_left -= 1 | ||||
if not retries_left: | if not retries_left: | ||||
logging.warning( | logger.warning( | ||||
'giving up on %s: max retries exceeded' % identifier) | 'giving up on %s: max retries exceeded' % identifier) | ||||
return r | return r | ||||
def db_query_equal(self, key, value): | def db_query_equal(self, key, value): | ||||
"""Look in the db for a row with key == value | """Look in the db for a row with key == value | ||||
Args: | Args: | ||||
▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines | def string_pattern_check(self, a, b, c=None): | ||||
pattern. | pattern. | ||||
""" | """ | ||||
if isinstance(a, str): | if isinstance(a, str): | ||||
a_pattern = re.sub('[a-zA-Z0-9]', | a_pattern = re.sub('[a-zA-Z0-9]', | ||||
'[a-zA-Z0-9]', | '[a-zA-Z0-9]', | ||||
re.escape(a)) | re.escape(a)) | ||||
if (isinstance(b, str) and (re.match(a_pattern, b) is None) | if (isinstance(b, str) and (re.match(a_pattern, b) is None) | ||||
or isinstance(c, str) and (re.match(a_pattern, c) is None)): | or isinstance(c, str) and (re.match(a_pattern, c) is None)): | ||||
logging.debug(a_pattern) | logger.debug(a_pattern) | ||||
raise TypeError('incomparable string patterns detected') | raise TypeError('incomparable string patterns detected') | ||||
def inject_repo_data_into_db(self, models_list): | def inject_repo_data_into_db(self, models_list): | ||||
"""Inject data into the db. | """Inject data into the db. | ||||
Args: | Args: | ||||
models_list: list of dicts mapping keys from the db model | models_list: list of dicts mapping keys from the db model | ||||
for each repo to be injected | for each repo to be injected | ||||
▲ Show 20 Lines • Show All 98 Lines • Show Last 20 Lines |