Page MenuHomeSoftware Heritage
Files F8396272

D4141.id14599.diff
No OneTemporary
Actions

D4141.id14599.diff
View Options

diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py
--- a/swh/lister/core/lister_base.py
+++ b/swh/lister/core/lister_base.py
@@ -37,6 +37,14 @@
return repr(self.response)
+DEFAULT_CONFIG = {
+ "scheduler": {"cls": "memory"},
+ "lister": {"cls": "local", "args": {"db": "postgresql:///lister",},},
+ "credentials": {},
+ "cache_responses": False,
+}
+
+
class ListerBase(abc.ABC, config.SWHConfig):
"""Lister core base class.
Generally a source code hosting service provides an API endpoint
@@ -214,39 +222,18 @@
# You probably don't need to override anything below this line.
- DEFAULT_CONFIG = {
- "scheduler": (
- "dict",
- {"cls": "remote", "args": {"url": "http://localhost:5008/"},},
- ),
- "lister": ("dict", {"cls": "local", "args": {"db": "postgresql:///lister",},}),
- }
-
- @property
- def CONFIG_BASE_FILENAME(self): # noqa: N802
- return "lister_%s" % self.LISTER_NAME
-
- @property
- def ADDITIONAL_CONFIG(self): # noqa: N802
- return {
- "credentials": ("dict", {}),
- "cache_responses": ("bool", False),
- "cache_dir": ("str", "~/.cache/swh/lister/%s" % self.LISTER_NAME),
- }
-
INITIAL_BACKOFF = 10
MAX_RETRIES = 7
CONN_SLEEP = 10
def __init__(self, override_config=None):
self.backoff = self.INITIAL_BACKOFF
- logger.debug("Loading config from %s" % self.CONFIG_BASE_FILENAME)
- self.config = self.parse_config_file(
- base_filename=self.CONFIG_BASE_FILENAME,
- additional_configs=[self.ADDITIONAL_CONFIG],
- )
- self.config["cache_dir"] = os.path.expanduser(self.config["cache_dir"])
+ self.config = config.load_from_envvar(DEFAULT_CONFIG)
if self.config["cache_responses"]:
+ cache_dir = self.config.get("cached_dir")
+ if not cache_dir:
+ cache_dir = f"~/.cache/swh/lister/{self.LISTER_NAME}"
+ self.config["cache_dir"] = os.path.expanduser(self.config["cache_dir"])
config.prepare_folders(self.config, "cache_dir")
if override_config:
diff --git a/swh/lister/npm/lister.py b/swh/lister/npm/lister.py
--- a/swh/lister/npm/lister.py
+++ b/swh/lister/npm/lister.py
@@ -6,10 +6,15 @@
from requests import Response
+from swh.core import config
from swh.lister.core.indexing_lister import IndexingHttpLister
from swh.lister.npm.models import NpmModel
from swh.scheduler.utils import create_task_dict
+DEFAULT_CONFIG = {
+ "loading_task_policy": "recurring",
+}
+
class NpmListerBase(IndexingHttpLister):
"""List packages available in the npm registry in a paginated way
@@ -24,18 +29,10 @@
self, url="https://replicate.npmjs.com", per_page=1000, override_config=None
):
super().__init__(url=url, override_config=override_config)
+ self.config = config.merge_configs(DEFAULT_CONFIG, self.config)
self.per_page = per_page + 1
self.PATH_TEMPLATE += "&limit=%s" % self.per_page
- @property
- def ADDITIONAL_CONFIG(self) -> Dict[str, Any]:
- """(Override) Add extra configuration
-
- """
- default_config = super().ADDITIONAL_CONFIG
- default_config["loading_task_policy"] = ("str", "recurring")
- return default_config
-
def get_model_from_repo(self, repo_name: str) -> Dict[str, str]:
"""(Override) Transform from npm package name to model
diff --git a/swh/lister/pytest_plugin.py b/swh/lister/pytest_plugin.py
--- a/swh/lister/pytest_plugin.py
+++ b/swh/lister/pytest_plugin.py
@@ -4,9 +4,11 @@
# See top-level LICENSE file for more information
import logging
+import os
import pytest
from sqlalchemy import create_engine
+import yaml
from swh.lister import SUPPORTED_LISTERS, get_lister
from swh.lister.core.models import initialize
@@ -33,7 +35,28 @@
@pytest.fixture
-def swh_lister(mock_get_scheduler, lister_db_url, swh_scheduler, lister_under_test):
+def swh_lister_config(lister_db_url, swh_scheduler_config):
+ return {
+ "scheduler": {"cls": "local", "args": {"db": swh_scheduler_config}["db"]},
+ "lister": {"cls": "local", "args": {"db": lister_db_url},},
+ "credentials": {},
+ "cache_responses": False,
+ }
+
+
+@pytest.fixture(autouse=True)
+def swh_config(swh_lister_config, monkeypatch, tmp_path):
+ conffile = os.path.join(str(tmp_path), "lister.yml")
+ with open(conffile, "w") as f:
+ f.write(yaml.dump(swh_lister_config))
+ monkeypatch.setenv("SWH_CONFIG_FILENAME", conffile)
+ return conffile
+
+
+@pytest.fixture
+def swh_lister(
+ mock_get_scheduler, lister_db_url, swh_scheduler, lister_under_test, swh_config
+):
assert lister_under_test in SUPPORTED_LISTERS
lister = get_lister(lister_under_test, db_url=lister_db_url)
initialize(create_engine(lister_db_url), drop_tables=True)

File Metadata

Mime Type
text/plain
Expires
Jun 3 2025, 7:51 PM (12 w, 3 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3224354

Event Timeline

About · Developer information · Legal