diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py --- a/swh/lister/core/lister_base.py +++ b/swh/lister/core/lister_base.py @@ -37,6 +37,14 @@ return repr(self.response) +DEFAULT_CONFIG = { + "scheduler": {"cls": "memory"}, + "lister": {"cls": "local", "args": {"db": "postgresql:///lister",},}, + "credentials": {}, + "cache_responses": False, +} + + class ListerBase(abc.ABC, config.SWHConfig): """Lister core base class. Generally a source code hosting service provides an API endpoint @@ -214,39 +222,16 @@ # You probably don't need to override anything below this line. - DEFAULT_CONFIG = { - "scheduler": ( - "dict", - {"cls": "remote", "args": {"url": "http://localhost:5008/"},}, - ), - "lister": ("dict", {"cls": "local", "args": {"db": "postgresql:///lister",},}), - } - - @property - def CONFIG_BASE_FILENAME(self): # noqa: N802 - return "lister_%s" % self.LISTER_NAME - - @property - def ADDITIONAL_CONFIG(self): # noqa: N802 - return { - "credentials": ("dict", {}), - "cache_responses": ("bool", False), - "cache_dir": ("str", "~/.cache/swh/lister/%s" % self.LISTER_NAME), - } - INITIAL_BACKOFF = 10 MAX_RETRIES = 7 CONN_SLEEP = 10 def __init__(self, override_config=None): self.backoff = self.INITIAL_BACKOFF - logger.debug("Loading config from %s" % self.CONFIG_BASE_FILENAME) - self.config = self.parse_config_file( - base_filename=self.CONFIG_BASE_FILENAME, - additional_configs=[self.ADDITIONAL_CONFIG], - ) - self.config["cache_dir"] = os.path.expanduser(self.config["cache_dir"]) + self.config = config.load_from_envvar(DEFAULT_CONFIG) if self.config["cache_responses"]: + cache_dir = self.config.get("cache_dir", f"~/.cache/swh/lister/{self.LISTER_NAME}") + self.config["cache_dir"] = os.path.expanduser(self.config["cache_dir"]) config.prepare_folders(self.config, "cache_dir") if override_config: diff --git a/swh/lister/npm/lister.py b/swh/lister/npm/lister.py --- a/swh/lister/npm/lister.py +++ b/swh/lister/npm/lister.py @@ -6,10 +6,15 @@ from requests import Response +from swh.core import config from swh.lister.core.indexing_lister import IndexingHttpLister from swh.lister.npm.models import NpmModel from swh.scheduler.utils import create_task_dict +DEFAULT_CONFIG = { + "loading_task_policy": "recurring", +} + class NpmListerBase(IndexingHttpLister): """List packages available in the npm registry in a paginated way @@ -24,18 +29,10 @@ self, url="https://replicate.npmjs.com", per_page=1000, override_config=None ): super().__init__(url=url, override_config=override_config) + self.config = config.merge_configs(DEFAULT_CONFIG, self.config) self.per_page = per_page + 1 self.PATH_TEMPLATE += "&limit=%s" % self.per_page - @property - def ADDITIONAL_CONFIG(self) -> Dict[str, Any]: - """(Override) Add extra configuration - - """ - default_config = super().ADDITIONAL_CONFIG - default_config["loading_task_policy"] = ("str", "recurring") - return default_config - def get_model_from_repo(self, repo_name: str) -> Dict[str, str]: """(Override) Transform from npm package name to model diff --git a/swh/lister/pytest_plugin.py b/swh/lister/pytest_plugin.py --- a/swh/lister/pytest_plugin.py +++ b/swh/lister/pytest_plugin.py @@ -4,9 +4,11 @@ # See top-level LICENSE file for more information import logging +import os import pytest from sqlalchemy import create_engine +import yaml from swh.lister import SUPPORTED_LISTERS, get_lister from swh.lister.core.models import initialize @@ -33,7 +35,28 @@ @pytest.fixture -def swh_lister(mock_get_scheduler, lister_db_url, swh_scheduler, lister_under_test): +def swh_lister_config(lister_db_url, swh_scheduler_config): + return { + "scheduler": {"cls": "local", "args": {"db": swh_scheduler_config}["db"]}, + "lister": {"cls": "local", "args": {"db": lister_db_url},}, + "credentials": {}, + "cache_responses": False, + } + + +@pytest.fixture(autouse=True) +def swh_config(swh_lister_config, monkeypatch, tmp_path): + conf_path = os.path.join(str(tmp_path), "lister.yml") + with open(conf_path, "w") as f: + f.write(yaml.dump(swh_lister_config)) + monkeypatch.setenv("SWH_CONFIG_FILENAME", conf_path) + return conf_path + + +@pytest.fixture +def swh_lister( + mock_get_scheduler, lister_db_url, swh_scheduler, lister_under_test, swh_config +): assert lister_under_test in SUPPORTED_LISTERS lister = get_lister(lister_under_test, db_url=lister_db_url) initialize(create_engine(lister_db_url), drop_tables=True)