Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/core/lister_base.py
| Show First 20 Lines • Show All 194 Lines • ▼ Show 20 Lines | def is_within_bounds(self, inner, lower=None, upper=None): | ||||
| ret = inner <= upper | ret = inner <= upper | ||||
| elif upper is None: | elif upper is None: | ||||
| ret = inner >= lower | ret = inner >= lower | ||||
| else: | else: | ||||
| ret = lower <= inner <= upper | ret = lower <= inner <= upper | ||||
| self.string_pattern_check(inner, lower, upper) | self.string_pattern_check(inner, lower, upper) | ||||
| except Exception as e: | except Exception as e: | ||||
| logging.error(str(e) + ': %s, %s, %s' % | logger.error(str(e) + ': %s, %s, %s' % | ||||
| (('inner=%s%s' % (type(inner), inner)), | (('inner=%s%s' % (type(inner), inner)), | ||||
| ('lower=%s%s' % (type(lower), lower)), | ('lower=%s%s' % (type(lower), lower)), | ||||
| ('upper=%s%s' % (type(upper), upper))) | ('upper=%s%s' % (type(upper), upper))) | ||||
| ) | ) | ||||
| raise | raise | ||||
| return ret | return ret | ||||
| # You probably don't need to override anything below this line. | # You probably don't need to override anything below this line. | ||||
| DEFAULT_CONFIG = { | DEFAULT_CONFIG = { | ||||
| 'storage': ('dict', { | 'storage': ('dict', { | ||||
| Show All 28 Lines | def ADDITIONAL_CONFIG(self): # noqa: N802 | ||||
| } | } | ||||
| INITIAL_BACKOFF = 10 | INITIAL_BACKOFF = 10 | ||||
| MAX_RETRIES = 7 | MAX_RETRIES = 7 | ||||
| CONN_SLEEP = 10 | CONN_SLEEP = 10 | ||||
| def __init__(self, override_config=None): | def __init__(self, override_config=None): | ||||
| self.backoff = self.INITIAL_BACKOFF | self.backoff = self.INITIAL_BACKOFF | ||||
| logging.debug('Loading config from %s' % self.CONFIG_BASE_FILENAME) | logger.debug('Loading config from %s' % self.CONFIG_BASE_FILENAME) | ||||
| self.config = self.parse_config_file( | self.config = self.parse_config_file( | ||||
| base_filename=self.CONFIG_BASE_FILENAME, | base_filename=self.CONFIG_BASE_FILENAME, | ||||
| additional_configs=[self.ADDITIONAL_CONFIG] | additional_configs=[self.ADDITIONAL_CONFIG] | ||||
| ) | ) | ||||
| self.config['cache_dir'] = os.path.expanduser(self.config['cache_dir']) | self.config['cache_dir'] = os.path.expanduser(self.config['cache_dir']) | ||||
| if self.config['cache_responses']: | if self.config['cache_responses']: | ||||
| config.prepare_folders(self.config, 'cache_dir') | config.prepare_folders(self.config, 'cache_dir') | ||||
| Show All 31 Lines | def safely_issue_request(self, identifier): | ||||
| retries_left = self.MAX_RETRIES | retries_left = self.MAX_RETRIES | ||||
| do_cache = self.config['cache_responses'] | do_cache = self.config['cache_responses'] | ||||
| r = None | r = None | ||||
| while retries_left > 0: | while retries_left > 0: | ||||
| try: | try: | ||||
| r = self.transport_request(identifier) | r = self.transport_request(identifier) | ||||
| except FetchError: | except FetchError: | ||||
| # network-level connection error, try again | # network-level connection error, try again | ||||
| logging.warning( | logger.warning( | ||||
| 'connection error on %s: sleep for %d seconds' % | 'connection error on %s: sleep for %d seconds' % | ||||
| (identifier, self.CONN_SLEEP)) | (identifier, self.CONN_SLEEP)) | ||||
| time.sleep(self.CONN_SLEEP) | time.sleep(self.CONN_SLEEP) | ||||
| retries_left -= 1 | retries_left -= 1 | ||||
| continue | continue | ||||
| if do_cache: | if do_cache: | ||||
| self.save_response(r) | self.save_response(r) | ||||
| # detect throttling | # detect throttling | ||||
| must_retry, delay = self.transport_quota_check(r) | must_retry, delay = self.transport_quota_check(r) | ||||
| if must_retry: | if must_retry: | ||||
| logging.warning( | logger.warning( | ||||
| 'rate limited on %s: sleep for %f seconds' % | 'rate limited on %s: sleep for %f seconds' % | ||||
| (identifier, delay)) | (identifier, delay)) | ||||
| time.sleep(delay) | time.sleep(delay) | ||||
| else: # request ok | else: # request ok | ||||
| break | break | ||||
| retries_left -= 1 | retries_left -= 1 | ||||
| if not retries_left: | if not retries_left: | ||||
| logging.warning( | logger.warning( | ||||
| 'giving up on %s: max retries exceeded' % identifier) | 'giving up on %s: max retries exceeded' % identifier) | ||||
| return r | return r | ||||
| def db_query_equal(self, key, value): | def db_query_equal(self, key, value): | ||||
| """Look in the db for a row with key == value | """Look in the db for a row with key == value | ||||
| Args: | Args: | ||||
| ▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines | def string_pattern_check(self, a, b, c=None): | ||||
| pattern. | pattern. | ||||
| """ | """ | ||||
| if isinstance(a, str): | if isinstance(a, str): | ||||
| a_pattern = re.sub('[a-zA-Z0-9]', | a_pattern = re.sub('[a-zA-Z0-9]', | ||||
| '[a-zA-Z0-9]', | '[a-zA-Z0-9]', | ||||
| re.escape(a)) | re.escape(a)) | ||||
| if (isinstance(b, str) and (re.match(a_pattern, b) is None) | if (isinstance(b, str) and (re.match(a_pattern, b) is None) | ||||
| or isinstance(c, str) and (re.match(a_pattern, c) is None)): | or isinstance(c, str) and (re.match(a_pattern, c) is None)): | ||||
| logging.debug(a_pattern) | logger.debug(a_pattern) | ||||
| raise TypeError('incomparable string patterns detected') | raise TypeError('incomparable string patterns detected') | ||||
| def inject_repo_data_into_db(self, models_list): | def inject_repo_data_into_db(self, models_list): | ||||
| """Inject data into the db. | """Inject data into the db. | ||||
| Args: | Args: | ||||
| models_list: list of dicts mapping keys from the db model | models_list: list of dicts mapping keys from the db model | ||||
| for each repo to be injected | for each repo to be injected | ||||
| ▲ Show 20 Lines • Show All 98 Lines • Show Last 20 Lines | |||||