diff --git a/swh/lister/__init__.py b/swh/lister/__init__.py --- a/swh/lister/__init__.py +++ b/swh/lister/__init__.py @@ -7,6 +7,8 @@ import pkg_resources +from swh.lister import pattern + logger = logging.getLogger(__name__) @@ -51,5 +53,8 @@ registry_entry = LISTERS[lister_name].load()() lister_cls = registry_entry["lister"] - lister = lister_cls(override_config=conf) - return lister + if issubclass(lister_cls, pattern.Lister): + return lister_cls.from_config(**conf) + else: + # Old-style lister + return lister_cls(override_config=conf) diff --git a/swh/lister/pattern.py b/swh/lister/pattern.py --- a/swh/lister/pattern.py +++ b/swh/lister/pattern.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Any, Dict, Generic, Iterable, Iterator, List, Optional, TypeVar -from swh.scheduler import model +from swh.scheduler import get_scheduler, model from swh.scheduler.interface import SchedulerInterface @@ -222,3 +222,27 @@ """ ret = self.scheduler.record_listed_origins(origins) return len(ret) + + @classmethod + def from_config(cls, scheduler: Dict[str, Any], **config: Any): + """Instantiate a lister from a configuration dict. + + This is basically a backwards-compatibility shim for the CLI. + + Args: + scheduler: instantiation config for the scheduler + config: the configuration dict for the lister, with the following keys: + - credentials (optional): credentials list for the scheduler + - any other kwargs passed to the lister. + + Returns: + the instantiated lister + """ + # Drop the legacy config keys which aren't used for this generation of listers. + for legacy_key in ("storage", "lister", "celery"): + config.pop(legacy_key, None) + + # Instantiate the scheduler + scheduler_instance = get_scheduler(**scheduler) + + return cls(scheduler=scheduler_instance, **config) diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py --- a/swh/lister/tests/test_cli.py +++ b/swh/lister/tests/test_cli.py @@ -6,7 +6,6 @@ import pytest from swh.lister.cli import SUPPORTED_LISTERS, get_lister -from swh.lister.core.lister_base import ListerBase from .test_utils import init_db @@ -19,17 +18,16 @@ assert "Invalid lister" in str(e.value) -def test_get_lister(): +def test_get_lister(swh_scheduler_config): """Instantiating a supported lister should be ok """ db_url = init_db().url() - # exclude listers because they need special instantiation treatment unrelated to - # this test (launchpad: network mock, gnu: scheduler load task) - listers_to_instantiate = set(SUPPORTED_LISTERS) - {"launchpad", "gnu"} - for lister_name in listers_to_instantiate: - lst = get_lister(lister_name, db_url) - assert isinstance(lst, ListerBase) + for lister_name in SUPPORTED_LISTERS: + lst = get_lister( + lister_name, db_url, scheduler={"cls": "local", **swh_scheduler_config} + ) + assert hasattr(lst, "run") def test_get_lister_override(): @@ -47,9 +45,7 @@ # check the override ends up defined in the lister for lister_name, url in listers.items(): lst = get_lister( - lister_name, - db_url, - **{"url": url, "priority": "high", "policy": "oneshot",}, + lister_name, db_url, url=url, priority="high", policy="oneshot" ) assert lst.url == url