Page MenuHomeSoftware Heritage
Files F8395810

D3526.id16314.diff
No OneTemporary
Actions

D3526.id16314.diff
View Options

diff --git a/swh/lister/__init__.py b/swh/lister/__init__.py
--- a/swh/lister/__init__.py
+++ b/swh/lister/__init__.py
@@ -7,6 +7,8 @@
import pkg_resources
+from swh.lister import pattern
+
logger = logging.getLogger(__name__)
@@ -51,5 +53,8 @@
registry_entry = LISTERS[lister_name].load()()
lister_cls = registry_entry["lister"]
- lister = lister_cls(override_config=conf)
- return lister
+ if issubclass(lister_cls, pattern.Lister):
+ return pattern.lister_from_config(lister_cls, **conf)
+ else:
+ # Old-style lister
+ return lister_cls(override_config=conf)
diff --git a/swh/lister/pattern.py b/swh/lister/pattern.py
--- a/swh/lister/pattern.py
+++ b/swh/lister/pattern.py
@@ -4,9 +4,9 @@
# See top-level LICENSE file for more information
from dataclasses import dataclass
-from typing import Any, Dict, Generic, Iterable, Iterator, List, Optional, TypeVar
+from typing import Any, Dict, Generic, Iterable, Iterator, List, Optional, Type, TypeVar
-from swh.scheduler import model
+from swh.scheduler import get_scheduler, model
from swh.scheduler.interface import SchedulerInterface
@@ -222,3 +222,28 @@
"""
ret = self.scheduler.record_listed_origins(origins)
return len(ret)
+
+
+def lister_from_config(lister_cls: Type[Lister], **config: Dict[str, Any]) -> Lister:
+ """Instantiate a lister from a configuration dict.
+
+ This is basically a backwards-compatibility shim for the CLI.
+
+ Args:
+ lister_cls: the class of lister to instantiate
+ config: the configuration dict for the lister, with the following keys:
+ - scheduler: instantiation config for the scheduler
+ - credentials (optional): credentials list for the scheduler
+ - any other kwargs passed to the lister.
+
+ Returns:
+ the instantiated lister
+ """
+ # Drop the legacy config keys which aren't used for this generation of listers.
+ for legacy_key in ("storage", "lister", "celery"):
+ config.pop(legacy_key, None)
+
+ # Instantiate the scheduler
+ scheduler = get_scheduler(**config.pop("scheduler"))
+
+ return lister_cls(scheduler=scheduler, **config)
diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py
--- a/swh/lister/tests/test_cli.py
+++ b/swh/lister/tests/test_cli.py
@@ -6,7 +6,6 @@
import pytest
from swh.lister.cli import SUPPORTED_LISTERS, get_lister
-from swh.lister.core.lister_base import ListerBase
from .test_utils import init_db
@@ -29,7 +28,7 @@
listers_to_instantiate = set(SUPPORTED_LISTERS) - {"launchpad", "gnu"}
for lister_name in listers_to_instantiate:
lst = get_lister(lister_name, db_url)
- assert isinstance(lst, ListerBase)
+ assert hasattr(lst, "run")
def test_get_lister_override():
@@ -47,9 +46,7 @@
# check the override ends up defined in the lister
for lister_name, url in listers.items():
lst = get_lister(
- lister_name,
- db_url,
- **{"url": url, "priority": "high", "policy": "oneshot",},
+ lister_name, db_url, url=url, priority="high", policy="oneshot"
)
assert lst.url == url

File Metadata

Mime Type
text/plain
Expires
Jun 3 2025, 7:45 PM (11 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3231442

Event Timeline

About · Developer information · Legal