Page MenuHomeSoftware Heritage
Files F7124651

D3526.diff
No OneTemporary
Actions

D3526.diff
View Options

diff --git a/swh/lister/__init__.py b/swh/lister/__init__.py
--- a/swh/lister/__init__.py
+++ b/swh/lister/__init__.py
@@ -7,6 +7,8 @@
import pkg_resources
+from swh.lister import pattern
+
logger = logging.getLogger(__name__)
@@ -51,5 +53,8 @@
registry_entry = LISTERS[lister_name].load()()
lister_cls = registry_entry["lister"]
- lister = lister_cls(override_config=conf)
- return lister
+ if issubclass(lister_cls, pattern.Lister):
+ return lister_cls.from_config(**conf)
+ else:
+ # Old-style lister
+ return lister_cls(override_config=conf)
diff --git a/swh/lister/pattern.py b/swh/lister/pattern.py
--- a/swh/lister/pattern.py
+++ b/swh/lister/pattern.py
@@ -6,7 +6,7 @@
from dataclasses import dataclass
from typing import Any, Dict, Generic, Iterable, Iterator, List, Optional, TypeVar
-from swh.scheduler import model
+from swh.scheduler import get_scheduler, model
from swh.scheduler.interface import SchedulerInterface
@@ -222,3 +222,27 @@
"""
ret = self.scheduler.record_listed_origins(origins)
return len(ret)
+
+ @classmethod
+ def from_config(cls, scheduler: Dict[str, Any], **config: Any):
+ """Instantiate a lister from a configuration dict.
+
+ This is basically a backwards-compatibility shim for the CLI.
+
+ Args:
+ scheduler: instantiation config for the scheduler
+ config: the configuration dict for the lister, with the following keys:
+ - credentials (optional): credentials list for the scheduler
+ - any other kwargs passed to the lister.
+
+ Returns:
+ the instantiated lister
+ """
+ # Drop the legacy config keys which aren't used for this generation of listers.
+ for legacy_key in ("storage", "lister", "celery"):
+ config.pop(legacy_key, None)
+
+ # Instantiate the scheduler
+ scheduler_instance = get_scheduler(**scheduler)
+
+ return cls(scheduler=scheduler_instance, **config)
diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py
--- a/swh/lister/tests/test_cli.py
+++ b/swh/lister/tests/test_cli.py
@@ -6,7 +6,6 @@
import pytest
from swh.lister.cli import SUPPORTED_LISTERS, get_lister
-from swh.lister.core.lister_base import ListerBase
from .test_utils import init_db
@@ -19,17 +18,16 @@
assert "Invalid lister" in str(e.value)
-def test_get_lister():
+def test_get_lister(swh_scheduler_config):
"""Instantiating a supported lister should be ok
"""
db_url = init_db().url()
- # exclude listers because they need special instantiation treatment unrelated to
- # this test (launchpad: network mock, gnu: scheduler load task)
- listers_to_instantiate = set(SUPPORTED_LISTERS) - {"launchpad", "gnu"}
- for lister_name in listers_to_instantiate:
- lst = get_lister(lister_name, db_url)
- assert isinstance(lst, ListerBase)
+ for lister_name in SUPPORTED_LISTERS:
+ lst = get_lister(
+ lister_name, db_url, scheduler={"cls": "local", **swh_scheduler_config}
+ )
+ assert hasattr(lst, "run")
def test_get_lister_override():
@@ -47,9 +45,7 @@
# check the override ends up defined in the lister
for lister_name, url in listers.items():
lst = get_lister(
- lister_name,
- db_url,
- **{"url": url, "priority": "high", "policy": "oneshot",},
+ lister_name, db_url, url=url, priority="high", policy="oneshot"
)
assert lst.url == url

File Metadata

Mime Type
text/plain
Expires
Dec 21 2024, 4:02 PM (11 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3219980

Event Timeline

About · Developer information · Legal