diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py --- a/swh/lister/core/lister_base.py +++ b/swh/lister/core/lister_base.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 the Software Heritage developers +# Copyright (C) 2015-2019 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -13,7 +13,7 @@ from sqlalchemy import create_engine, func from sqlalchemy.orm import sessionmaker -from typing import Any, Type, Union +from typing import Any, Sequence, Type, Union from swh.core import config from swh.scheduler import get_scheduler, utils @@ -71,6 +71,8 @@ ) # type: Union[AbstractAttribute, Type[Any]] LISTER_NAME = AbstractAttribute( "Lister's name") # type: Union[AbstractAttribute, str] + # Lister's output scheduler task types (default to empty) + SCHEDULED_TASK_TYPES: Sequence[str] = [] def transport_request(self, identifier): """Given a target endpoint identifier to query, try once to request it. @@ -257,6 +259,24 @@ self.mk_session = sessionmaker(bind=self.db_engine) self.db_session = self.mk_session() + def checks(self): + """Basic checks on lister configuration. + + This checks the expected output task types exist in the scheduler. If + it does not, this raises a ValueError. + + """ + missing_types = [] + for task_type in self.SCHEDULED_TASK_TYPES: + ok = self.scheduler.get_task_type(task_type) + if not ok: + missing_types.append(task_type) + + if missing_types: + raise ValueError( + "Misconfigured lister: Missing %s type in scheduler" % ( + ','.join(missing_types))) + def reset_backoff(self): """Reset exponential backoff timeout to initial level.""" self.backoff = self.INITIAL_BACKOFF diff --git a/swh/lister/debian/lister.py b/swh/lister/debian/lister.py --- a/swh/lister/debian/lister.py +++ b/swh/lister/debian/lister.py @@ -34,10 +34,14 @@ class DebianLister(ListerHttpTransport, ListerBase): + """Debian distribution Lister. + + """ MODEL = Package PATH_TEMPLATE = None LISTER_NAME = 'debian' instance = 'debian' + SCHEDULED_TASK_TYPES = ['load-deb-package'] def __init__(self, distribution: str = 'Debian', date: Optional[datetime.datetime] = None, @@ -51,6 +55,9 @@ override_config: Override configuration (which takes precedence over the parameters if provided) + Output: + scheduler tasks of type 'load-deb-package' + """ ListerHttpTransport.__init__(self, url="notused") ListerBase.__init__(self, override_config=override_config) diff --git a/swh/lister/debian/tests/conftest.py b/swh/lister/debian/tests/conftest.py --- a/swh/lister/debian/tests/conftest.py +++ b/swh/lister/debian/tests/conftest.py @@ -14,11 +14,15 @@ def lister_debian(swh_listers): lister = swh_listers['debian'] - # Initialize the debian data model + # Initialize debian data model debian_init(lister.db_engine, distributions=['stretch'], area_names=['main', 'contrib']) + with pytest.raises(ValueError, + match='Misconfigured lister'): + lister.checks() + # Add the load-deb-package in the scheduler backend lister.scheduler.create_task_type({ 'type': 'load-deb-package', @@ -27,4 +31,7 @@ 'default_interval': '1 day', }) + # everything is now fine + lister.checks() + return lister