Page MenuHomeSoftware Heritage
Differential D1250

journal-publisher: Update the journal publisher entrypoint
ClosedPublic
Actions

Authored by ardumont on Mar 15 2019, 12:20 PM.

Details

Reviewers
vlorentz
Group Reviewers
Reviewers
Commits
rDENVc5a48242d23b: journal-publisher: Update the journal publisher entrypoint
rCDFDc5a48242d23b: journal-publisher: Update the journal publisher entrypoint
Summary

As the configuration is no longer implicit, add the missing bits and
pieces.

Related D1249
Related T1276

Diff Detail

Repository
rCDFD Dockerfiles for developers
Branch
master
Lint
No Linters Available
Unit
No Unit Test Coverage
Build Status
Buildable 4657
Build 6189: arc lint + arc unit

Revision Contents
Changeset List

PathSize
conf/
17 lines
4 lines
services/
swh-journal-publisher/
4 lines

Diff 3942

View Options

conf/journal_publisher.yml

Show First 20 Lines • Show All 58 Lines • ▼ Show 20 Lines """Lister core base class.
Optional Overrides: Optional Overrides:
def filter_before_inject def filter_before_inject
def is_within_bounds def is_within_bounds
""" """
MODEL = AbstractAttribute('Subclass type (not instance)' MODEL = AbstractAttribute('Subclass type (not instance)'
' of swh.lister.core.models.ModelBase' ' of swh.lister.core.models.ModelBase'
' customized for a specific service.') ' customized for a specific service.')
LISTER_NAME = AbstractAttribute("Lister's name")
@abc.abstractmethod @abc.abstractmethod
def transport_request(self, identifier): def transport_request(self, identifier):
"""Given a target endpoint identifier to query, try once to request it. """Given a target endpoint identifier to query, try once to request it.
Implementation of this method determines the network request protocol. Implementation of this method determines the network request protocol.
Args: Args:
▲ Show 20 Lines • Show All 119 Lines • ▼ Show 20 Lines DEFAULT_CONFIG = {
'url': 'http://localhost:5002/' 'url': 'http://localhost:5002/'
}, },
}), }),
'scheduling_db': ('str', 'dbname=softwareheritage-scheduler-dev'), 'scheduling_db': ('str', 'dbname=softwareheritage-scheduler-dev'),
} }
@property @property
def CONFIG_BASE_FILENAME(self): # noqa: N802 def CONFIG_BASE_FILENAME(self): # noqa: N802
return 'lister-%s' % self.lister_name return 'lister-%s' % self.LISTER_NAME
@property @property
def ADDITIONAL_CONFIG(self): # noqa: N802 def ADDITIONAL_CONFIG(self): # noqa: N802
return { return {
'lister_db_url': 'lister_db_url':
('str', 'postgresql:///lister-%s' % self.lister_name), ('str', 'postgresql:///lister-%s' % self.LISTER_NAME),
'credentials': 'credentials':
('list[dict]', []), ('list[dict]', []),
'cache_responses': 'cache_responses':
('bool', False), ('bool', False),
'cache_dir': 'cache_dir':
('str', '~/.cache/swh/lister/%s' % self.lister_name), ('str', '~/.cache/swh/lister/%s' % self.LISTER_NAME),
} }
INITIAL_BACKOFF = 10 INITIAL_BACKOFF = 10
MAX_RETRIES = 7 MAX_RETRIES = 7
CONN_SLEEP = 10 CONN_SLEEP = 10
def __init__(self, lister_name=None, override_config=None): def __init__(self, override_config=None):
self.backoff = self.INITIAL_BACKOFF self.backoff = self.INITIAL_BACKOFF
if lister_name is None:
raise NameError("Every lister must be assigned a lister_name.")
self.lister_name = lister_name # 'github?', 'bitbucket?', 'foo.com?'
self.config = self.parse_config_file( self.config = self.parse_config_file(
base_filename=self.CONFIG_BASE_FILENAME, base_filename=self.CONFIG_BASE_FILENAME,
additional_configs=[self.ADDITIONAL_CONFIG] additional_configs=[self.ADDITIONAL_CONFIG]
) )
self.config['cache_dir'] = os.path.expanduser(self.config['cache_dir']) self.config['cache_dir'] = os.path.expanduser(self.config['cache_dir'])
if self.config['cache_responses']: if self.config['cache_responses']:
config.prepare_folders(self.config, 'cache_dir') config.prepare_folders(self.config, 'cache_dir')
▲ Show 20 Lines • Show All 263 Lines • Show Last 20 Lines
View Options

docker-compose.yml

Show First 20 Lines • Show All 58 Lines • ▼ Show 20 Lines """Lister core base class.
Optional Overrides: Optional Overrides:
def filter_before_inject def filter_before_inject
def is_within_bounds def is_within_bounds
""" """
MODEL = AbstractAttribute('Subclass type (not instance)' MODEL = AbstractAttribute('Subclass type (not instance)'
' of swh.lister.core.models.ModelBase' ' of swh.lister.core.models.ModelBase'
' customized for a specific service.') ' customized for a specific service.')
LISTER_NAME = AbstractAttribute("Lister's name")
@abc.abstractmethod @abc.abstractmethod
def transport_request(self, identifier): def transport_request(self, identifier):
"""Given a target endpoint identifier to query, try once to request it. """Given a target endpoint identifier to query, try once to request it.
Implementation of this method determines the network request protocol. Implementation of this method determines the network request protocol.
Args: Args:
▲ Show 20 Lines • Show All 119 Lines • ▼ Show 20 Lines DEFAULT_CONFIG = {
'url': 'http://localhost:5002/' 'url': 'http://localhost:5002/'
}, },
}), }),
'scheduling_db': ('str', 'dbname=softwareheritage-scheduler-dev'), 'scheduling_db': ('str', 'dbname=softwareheritage-scheduler-dev'),
} }
@property @property
def CONFIG_BASE_FILENAME(self): # noqa: N802 def CONFIG_BASE_FILENAME(self): # noqa: N802
return 'lister-%s' % self.lister_name return 'lister-%s' % self.LISTER_NAME
@property @property
def ADDITIONAL_CONFIG(self): # noqa: N802 def ADDITIONAL_CONFIG(self): # noqa: N802
return { return {
'lister_db_url': 'lister_db_url':
('str', 'postgresql:///lister-%s' % self.lister_name), ('str', 'postgresql:///lister-%s' % self.LISTER_NAME),
'credentials': 'credentials':
('list[dict]', []), ('list[dict]', []),
'cache_responses': 'cache_responses':
('bool', False), ('bool', False),
'cache_dir': 'cache_dir':
('str', '~/.cache/swh/lister/%s' % self.lister_name), ('str', '~/.cache/swh/lister/%s' % self.LISTER_NAME),
} }
INITIAL_BACKOFF = 10 INITIAL_BACKOFF = 10
MAX_RETRIES = 7 MAX_RETRIES = 7
CONN_SLEEP = 10 CONN_SLEEP = 10
def __init__(self, lister_name=None, override_config=None): def __init__(self, override_config=None):
self.backoff = self.INITIAL_BACKOFF self.backoff = self.INITIAL_BACKOFF
if lister_name is None:
raise NameError("Every lister must be assigned a lister_name.")
self.lister_name = lister_name # 'github?', 'bitbucket?', 'foo.com?'
self.config = self.parse_config_file( self.config = self.parse_config_file(
base_filename=self.CONFIG_BASE_FILENAME, base_filename=self.CONFIG_BASE_FILENAME,
additional_configs=[self.ADDITIONAL_CONFIG] additional_configs=[self.ADDITIONAL_CONFIG]
) )
self.config['cache_dir'] = os.path.expanduser(self.config['cache_dir']) self.config['cache_dir'] = os.path.expanduser(self.config['cache_dir'])
if self.config['cache_responses']: if self.config['cache_responses']:
config.prepare_folders(self.config, 'cache_dir') config.prepare_folders(self.config, 'cache_dir')
▲ Show 20 Lines • Show All 263 Lines • Show Last 20 Lines
View Options

services/swh-journal-publisher/entrypoint.sh

Show First 20 Lines • Show All 58 Lines • ▼ Show 20 Lines """Lister core base class.
Optional Overrides: Optional Overrides:
def filter_before_inject def filter_before_inject
def is_within_bounds def is_within_bounds
""" """
MODEL = AbstractAttribute('Subclass type (not instance)' MODEL = AbstractAttribute('Subclass type (not instance)'
' of swh.lister.core.models.ModelBase' ' of swh.lister.core.models.ModelBase'
' customized for a specific service.') ' customized for a specific service.')
LISTER_NAME = AbstractAttribute("Lister's name")
@abc.abstractmethod @abc.abstractmethod
def transport_request(self, identifier): def transport_request(self, identifier):
"""Given a target endpoint identifier to query, try once to request it. """Given a target endpoint identifier to query, try once to request it.
Implementation of this method determines the network request protocol. Implementation of this method determines the network request protocol.
Args: Args:
▲ Show 20 Lines • Show All 119 Lines • ▼ Show 20 Lines DEFAULT_CONFIG = {
'url': 'http://localhost:5002/' 'url': 'http://localhost:5002/'
}, },
}), }),
'scheduling_db': ('str', 'dbname=softwareheritage-scheduler-dev'), 'scheduling_db': ('str', 'dbname=softwareheritage-scheduler-dev'),
} }
@property @property
def CONFIG_BASE_FILENAME(self): # noqa: N802 def CONFIG_BASE_FILENAME(self): # noqa: N802
return 'lister-%s' % self.lister_name return 'lister-%s' % self.LISTER_NAME
@property @property
def ADDITIONAL_CONFIG(self): # noqa: N802 def ADDITIONAL_CONFIG(self): # noqa: N802
return { return {
'lister_db_url': 'lister_db_url':
('str', 'postgresql:///lister-%s' % self.lister_name), ('str', 'postgresql:///lister-%s' % self.LISTER_NAME),
'credentials': 'credentials':
('list[dict]', []), ('list[dict]', []),
'cache_responses': 'cache_responses':
('bool', False), ('bool', False),
'cache_dir': 'cache_dir':
('str', '~/.cache/swh/lister/%s' % self.lister_name), ('str', '~/.cache/swh/lister/%s' % self.LISTER_NAME),
} }
INITIAL_BACKOFF = 10 INITIAL_BACKOFF = 10
MAX_RETRIES = 7 MAX_RETRIES = 7
CONN_SLEEP = 10 CONN_SLEEP = 10
def __init__(self, lister_name=None, override_config=None): def __init__(self, override_config=None):
self.backoff = self.INITIAL_BACKOFF self.backoff = self.INITIAL_BACKOFF
if lister_name is None:
raise NameError("Every lister must be assigned a lister_name.")
self.lister_name = lister_name # 'github?', 'bitbucket?', 'foo.com?'
self.config = self.parse_config_file( self.config = self.parse_config_file(
base_filename=self.CONFIG_BASE_FILENAME, base_filename=self.CONFIG_BASE_FILENAME,
additional_configs=[self.ADDITIONAL_CONFIG] additional_configs=[self.ADDITIONAL_CONFIG]
) )
self.config['cache_dir'] = os.path.expanduser(self.config['cache_dir']) self.config['cache_dir'] = os.path.expanduser(self.config['cache_dir'])
if self.config['cache_responses']: if self.config['cache_responses']:
config.prepare_folders(self.config, 'cache_dir') config.prepare_folders(self.config, 'cache_dir')
▲ Show 20 Lines • Show All 263 Lines • Show Last 20 Lines
About · Developer information · Legal