diff --git a/swh/lister/__init__.py b/swh/lister/__init__.py --- a/swh/lister/__init__.py +++ b/swh/lister/__init__.py @@ -7,6 +7,8 @@ import pkg_resources +from swh.lister import pattern + logger = logging.getLogger(__name__) @@ -51,5 +53,8 @@ registry_entry = LISTERS[lister_name].load()() lister_cls = registry_entry["lister"] - lister = lister_cls(override_config=conf) - return lister + if issubclass(lister_cls, pattern.Lister): + return pattern.lister_from_config(lister_cls, **conf) + else: + # Old-style lister + return lister_cls(override_config=conf) diff --git a/swh/lister/pattern.py b/swh/lister/pattern.py --- a/swh/lister/pattern.py +++ b/swh/lister/pattern.py @@ -4,9 +4,9 @@ # See top-level LICENSE file for more information from dataclasses import dataclass -from typing import Any, Dict, Generic, Iterable, Iterator, List, Optional, TypeVar +from typing import Any, Dict, Generic, Iterable, Iterator, List, Optional, Type, TypeVar -from swh.scheduler import model +from swh.scheduler import get_scheduler, model from swh.scheduler.interface import SchedulerInterface @@ -222,3 +222,28 @@ """ ret = self.scheduler.record_listed_origins(origins) return len(ret) + + +def lister_from_config(lister_cls: Type[Lister], **config: Dict[str, Any]) -> Lister: + """Instantiate a lister from a configuration dict. + + This is basically a backwards-compatibility shim for the CLI. + + Args: + lister_cls: the class of lister to instantiate + config: the configuration dict for the lister, with the following keys: + - scheduler: instantiation config for the scheduler + - credentials (optional): credentials list for the scheduler + - any other kwargs passed to the lister. + + Returns: + the instantiated lister + """ + # Drop the legacy config keys which aren't used for this generation of listers. + for legacy_key in ("storage", "lister", "celery"): + config.pop(legacy_key, None) + + # Instantiate the scheduler + scheduler = get_scheduler(**config.pop("scheduler")) + + return lister_cls(scheduler=scheduler, **config) diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py --- a/swh/lister/tests/test_cli.py +++ b/swh/lister/tests/test_cli.py @@ -6,7 +6,6 @@ import pytest from swh.lister.cli import SUPPORTED_LISTERS, get_lister -from swh.lister.core.lister_base import ListerBase from .test_utils import init_db @@ -29,7 +28,7 @@ listers_to_instantiate = set(SUPPORTED_LISTERS) - {"launchpad", "gnu"} for lister_name in listers_to_instantiate: lst = get_lister(lister_name, db_url) - assert isinstance(lst, ListerBase) + assert hasattr(lst, "run") def test_get_lister_override(): @@ -47,9 +46,7 @@ # check the override ends up defined in the lister for lister_name, url in listers.items(): lst = get_lister( - lister_name, - db_url, - **{"url": url, "priority": "high", "policy": "oneshot",}, + lister_name, db_url, url=url, priority="high", policy="oneshot" ) assert lst.url == url