Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/core/lister_base.py
| Show First 20 Lines • Show All 233 Lines • ▼ Show 20 Lines | class SWHListerBase(abc.ABC, config.SWHConfig): | ||||
| @property | @property | ||||
| def CONFIG_BASE_FILENAME(self): # noqa: N802 | def CONFIG_BASE_FILENAME(self): # noqa: N802 | ||||
| return 'lister_%s' % self.LISTER_NAME | return 'lister_%s' % self.LISTER_NAME | ||||
| @property | @property | ||||
| def ADDITIONAL_CONFIG(self): # noqa: N802 | def ADDITIONAL_CONFIG(self): # noqa: N802 | ||||
| return { | return { | ||||
| 'credentials': | 'credentials': ('dict', {}), | ||||
| ('list[dict]', []), | 'cache_responses': ('bool', False), | ||||
| 'cache_responses': | 'cache_dir': ('str', '~/.cache/swh/lister/%s' % self.LISTER_NAME), | ||||
| ('bool', False), | |||||
| 'cache_dir': | |||||
| ('str', '~/.cache/swh/lister/%s' % self.LISTER_NAME), | |||||
| } | } | ||||
| INITIAL_BACKOFF = 10 | INITIAL_BACKOFF = 10 | ||||
| MAX_RETRIES = 7 | MAX_RETRIES = 7 | ||||
| CONN_SLEEP = 10 | CONN_SLEEP = 10 | ||||
| def __init__(self, override_config=None): | def __init__(self, override_config=None): | ||||
| self.backoff = self.INITIAL_BACKOFF | self.backoff = self.INITIAL_BACKOFF | ||||
| ▲ Show 20 Lines • Show All 153 Lines • ▼ Show 20 Lines | def task_dict(self, origin_type, origin_url, **kwargs): | ||||
| """Return special dict format for the tasks list | """Return special dict format for the tasks list | ||||
| Args: | Args: | ||||
| origin_type (string) | origin_type (string) | ||||
| origin_url (string) | origin_url (string) | ||||
| Returns: | Returns: | ||||
| the same information in a different form | the same information in a different form | ||||
| """ | """ | ||||
| _type = 'origin-update-%s' % origin_type | _type = 'load-%s' % origin_type | ||||
| _policy = 'recurring' | _policy = 'recurring' | ||||
| return utils.create_task_dict(_type, _policy, origin_url) | return utils.create_task_dict(_type, _policy, origin_url) | ||||
| def string_pattern_check(self, a, b, c=None): | def string_pattern_check(self, a, b, c=None): | ||||
| """When comparing indexable types in is_within_bounds, complex strings | """When comparing indexable types in is_within_bounds, complex strings | ||||
| may not be allowed to differ in basic structure. If they do, it | may not be allowed to differ in basic structure. If they do, it | ||||
| could be a sign of not understanding the data well. For instance, | could be a sign of not understanding the data well. For instance, | ||||
| an ISO 8601 time string cannot be compared against its urlencoded | an ISO 8601 time string cannot be compared against its urlencoded | ||||
| ▲ Show 20 Lines • Show All 127 Lines • Show Last 20 Lines | |||||