diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py index 191b7c5..3757dc8 100644 --- a/swh/lister/bitbucket/tests/test_lister.py +++ b/swh/lister/bitbucket/tests/test_lister.py @@ -1,120 +1,126 @@ -# Copyright (C) 2017-2019 The Software Heritage developers +# Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import re import unittest from datetime import timedelta from urllib.parse import unquote import iso8601 +import pytest import requests_mock from swh.lister.bitbucket.lister import BitBucketLister from swh.lister.core.tests.test_lister import HttpListerTester +@pytest.fixture +def lister_under_test(): + return "bitbucket" + + def _convert_type(req_index): """Convert the req_index to its right type according to the model's "indexable" column. """ return iso8601.parse_date(unquote(req_index)) class BitBucketListerTester(HttpListerTester, unittest.TestCase): Lister = BitBucketLister test_re = re.compile(r"/repositories\?after=([^?&]+)") lister_subdir = "bitbucket" good_api_response_file = "data/https_api.bitbucket.org/response.json" bad_api_response_file = "data/https_api.bitbucket.org/empty_response.json" first_index = _convert_type("2008-07-12T07:44:01.476818+00:00") last_index = _convert_type("2008-07-19T06:16:43.044743+00:00") entries_per_page = 10 convert_type = _convert_type def request_index(self, request): """(Override) This is needed to emulate the listing bootstrap when no min_bound is provided to run """ m = self.test_re.search(request.path_url) idx = _convert_type(m.group(1)) if idx == self.Lister.default_min_bound: idx = self.first_index return idx @requests_mock.Mocker() def test_fetch_none_nodb(self, http_mocker): """Overridden because index is not an integer nor a string """ http_mocker.get(self.test_re, text=self.mock_response) fl = self.get_fl() self.disable_scheduler(fl) self.disable_db(fl) # stores no results fl.run( min_bound=self.first_index - timedelta(days=3), max_bound=self.first_index ) def test_is_within_bounds(self): fl = self.get_fl() self.assertTrue( fl.is_within_bounds( iso8601.parse_date("2008-07-15"), self.first_index, self.last_index ) ) self.assertFalse( fl.is_within_bounds( iso8601.parse_date("2008-07-20"), self.first_index, self.last_index ) ) self.assertFalse( fl.is_within_bounds( iso8601.parse_date("2008-07-11"), self.first_index, self.last_index ) ) -def test_lister_bitbucket(swh_listers, requests_mock_datadir): +def test_lister_bitbucket(swh_lister, requests_mock_datadir): """Simple bitbucket listing should create scheduled tasks (git, hg) """ - lister = swh_listers["bitbucket"] + lister = swh_lister lister.run() r = lister.scheduler.search_tasks(task_type="load-hg") assert len(r) == 9 for row in r: args = row["arguments"]["args"] kwargs = row["arguments"]["kwargs"] assert len(args) == 0 assert len(kwargs) == 1 url = kwargs["url"] assert url.startswith("https://bitbucket.org") assert row["policy"] == "recurring" assert row["priority"] is None r = lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 1 for row in r: args = row["arguments"]["args"] kwargs = row["arguments"]["kwargs"] assert len(args) == 0 assert len(kwargs) == 1 url = kwargs["url"] assert url.startswith("https://bitbucket.org") assert row["policy"] == "recurring" assert row["priority"] is None diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py index bc71e15..bed9096 100644 --- a/swh/lister/cgit/tests/test_lister.py +++ b/swh/lister/cgit/tests/test_lister.py @@ -1,82 +1,88 @@ -# Copyright (C) 2019 the Software Heritage developers +# Copyright (C) 2019-2020 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import pytest from swh.lister import __version__ -def test_lister_no_page(requests_mock_datadir, swh_listers): - lister = swh_listers["cgit"] +@pytest.fixture +def lister_under_test(): + return "cgit" + + +def test_lister_no_page(requests_mock_datadir, swh_lister): + lister = swh_lister assert lister.url == "https://git.savannah.gnu.org/cgit/" repos = list(lister.get_repos()) assert len(repos) == 977 assert repos[0] == "https://git.savannah.gnu.org/cgit/elisp-es.git/" # note the url below is NOT a subpath of /cgit/ assert repos[-1] == "https://git.savannah.gnu.org/path/to/yetris.git/" # noqa # note the url below is NOT on the same server assert repos[-2] == "http://example.org/cgit/xstarcastle.git/" -def test_lister_model(requests_mock_datadir, swh_listers): - lister = swh_listers["cgit"] +def test_lister_model(requests_mock_datadir, swh_lister): + lister = swh_lister repo = next(lister.get_repos()) model = lister.build_model(repo) assert model == { "uid": "https://git.savannah.gnu.org/cgit/elisp-es.git/", "name": "elisp-es.git", "origin_type": "git", "instance": "git.savannah.gnu.org", "origin_url": "https://git.savannah.gnu.org/git/elisp-es.git", } -def test_lister_with_pages(requests_mock_datadir, swh_listers): - lister = swh_listers["cgit"] +def test_lister_with_pages(requests_mock_datadir, swh_lister): + lister = swh_lister lister.url = "https://git.tizen/cgit/" repos = list(lister.get_repos()) # we should have 16 repos (listed on 3 pages) assert len(repos) == 16 -def test_lister_run(requests_mock_datadir, swh_listers): - lister = swh_listers["cgit"] +def test_lister_run(requests_mock_datadir, swh_lister): + lister = swh_lister lister.url = "https://git.tizen/cgit/" lister.run() r = lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 16 for row in r: assert row["type"] == "load-git" # arguments check args = row["arguments"]["args"] assert len(args) == 0 # kwargs kwargs = row["arguments"]["kwargs"] assert len(kwargs) == 1 url = kwargs["url"] assert url.startswith("https://git.tizen") assert row["policy"] == "recurring" assert row["priority"] is None -def test_lister_requests(requests_mock_datadir, swh_listers): - lister = swh_listers["cgit"] +def test_lister_requests(requests_mock_datadir, swh_lister): + lister = swh_lister lister.url = "https://git.tizen/cgit/" lister.run() assert len(requests_mock_datadir.request_history) != 0 for request in requests_mock_datadir.request_history: assert "User-Agent" in request.headers user_agent = request.headers["User-Agent"] assert "Software Heritage Lister" in user_agent assert __version__ in user_agent diff --git a/swh/lister/cran/tests/conftest.py b/swh/lister/cran/tests/conftest.py index 7b7777a..4f086d8 100644 --- a/swh/lister/cran/tests/conftest.py +++ b/swh/lister/cran/tests/conftest.py @@ -1,23 +1,26 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest @pytest.fixture -def lister_cran(swh_listers): - lister = swh_listers["cran"] +def lister_under_test(): + return "cran" - # Add the load-deb-package in the scheduler backend - lister.scheduler.create_task_type( + +@pytest.fixture +def lister_cran(swh_lister): + # Add the load-cran in the scheduler backend + swh_lister.scheduler.create_task_type( { "type": "load-cran", "description": "Load a CRAN package", "backend_name": "swh.loader.package.cran.tasks.LoaderCRAN", "default_interval": "1 day", } ) - return lister + return swh_lister diff --git a/swh/lister/debian/tests/conftest.py b/swh/lister/debian/tests/conftest.py index bea3f46..d624f8d 100644 --- a/swh/lister/debian/tests/conftest.py +++ b/swh/lister/debian/tests/conftest.py @@ -1,57 +1,62 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from pytest_postgresql.janitor import DatabaseJanitor from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from swh.lister.core.models import SQLBase from swh.lister.debian import debian_init @pytest.fixture -def lister_debian(swh_listers): - lister = swh_listers["debian"] +def lister_under_test(): + return "debian" + +@pytest.fixture +def lister_debian(swh_lister): # Initialize the debian data model - debian_init(lister.db_engine, suites=["stretch"], components=["main", "contrib"]) + debian_init( + swh_lister.db_engine, suites=["stretch"], components=["main", "contrib"] + ) # Add the load-deb-package in the scheduler backend - lister.scheduler.create_task_type( + swh_lister.scheduler.create_task_type( { "type": "load-deb-package", "description": "Load a Debian package", "backend_name": "swh.loader.debian.tasks.LoaderDebianPackage", "default_interval": "1 day", } ) - return lister + return swh_lister @pytest.fixture def sqlalchemy_engine(postgresql_proc): pg_host = postgresql_proc.host pg_port = postgresql_proc.port pg_user = postgresql_proc.user pg_db = "sqlalchemy-tests" url = f"postgresql://{pg_user}@{pg_host}:{pg_port}/{pg_db}" with DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, postgresql_proc.version): engine = create_engine(url) yield engine engine.dispose() @pytest.fixture def session(sqlalchemy_engine): SQLBase.metadata.create_all(sqlalchemy_engine) Session = sessionmaker(bind=sqlalchemy_engine) session = Session() yield session session.close() diff --git a/swh/lister/gitea/tests/test_lister.py b/swh/lister/gitea/tests/test_lister.py index 6bd5c1f..74161d2 100644 --- a/swh/lister/gitea/tests/test_lister.py +++ b/swh/lister/gitea/tests/test_lister.py @@ -1,60 +1,67 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import re import unittest +import pytest + from swh.lister.core.tests.test_lister import HttpListerTesterBase from swh.lister.gitea.lister import GiteaLister logger = logging.getLogger(__name__) class GiteaListerTester(HttpListerTesterBase, unittest.TestCase): Lister = GiteaLister test_re = re.compile(r"^.*/projects.*page=(\d+).*") lister_subdir = "gitea" good_api_response_file = "data/https_try.gitea.io/api_response.json" bad_api_response_file = "data/https_try.gitea.io/api_empty_response.json" first_index = 1 last_index = 2 entries_per_page = 3 convert_type = int def response_headers(self, request): headers = {} if self.request_index(request) == self.first_index: headers.update( { "Link": ";" ' rel="next"' % self.last_index } ) return headers -def test_lister_gitea(swh_listers, requests_mock_datadir): - lister: GiteaLister = swh_listers["gitea"] +@pytest.fixture +def lister_under_test(): + return "gitea" + + +def test_lister_gitea(swh_lister, requests_mock_datadir): + lister: GiteaLister = swh_lister lister.run() r = lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 3 for row in r: assert row["type"] == "load-git" # arguments check args = row["arguments"]["args"] assert len(args) == 0 # kwargs kwargs = row["arguments"]["kwargs"] url = kwargs["url"] assert url.startswith("https://try.gitea.io") assert row["policy"] == "recurring" assert row["priority"] is None diff --git a/swh/lister/github/tests/test_lister.py b/swh/lister/github/tests/test_lister.py index f33d721..fef8810 100644 --- a/swh/lister/github/tests/test_lister.py +++ b/swh/lister/github/tests/test_lister.py @@ -1,83 +1,87 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import re import unittest +import pytest import requests_mock from swh.lister.core.tests.test_lister import HttpListerTester from swh.lister.github.lister import GitHubLister class GitHubListerTester(HttpListerTester, unittest.TestCase): Lister = GitHubLister test_re = re.compile(r"/repositories\?since=([^?&]+)") lister_subdir = "github" good_api_response_file = "data/https_api.github.com/first_response.json" bad_api_response_file = "data/https_api.github.com/empty_response.json" first_index = 0 last_index = 369 entries_per_page = 100 convert_type = int def response_headers(self, request): headers = {"X-RateLimit-Remaining": "1"} if self.request_index(request) == self.first_index: headers.update( { "Link": ";" ' rel="next",' ";" ' rel="first"' % self.last_index } ) else: headers.update( { "Link": ";" ' rel="first"' } ) return headers def mock_rate_quota(self, n, request, context): self.rate_limit += 1 context.status_code = 403 context.headers["X-RateLimit-Remaining"] = "0" context.headers["Retry-After"] = "1" # 1 second return '{"error":"dummy"}' @requests_mock.Mocker() def test_scheduled_tasks(self, http_mocker): self.scheduled_tasks_test( "data/https_api.github.com/next_response.json", 876, http_mocker ) -def test_lister_github(swh_listers, requests_mock_datadir): +@pytest.fixture +def lister_under_test(): + return "github" + + +def test_lister_github(swh_lister, requests_mock_datadir): """Simple github listing should create scheduled tasks """ - lister = swh_listers["github"] - - lister.run() + swh_lister.run() - r = lister.scheduler.search_tasks(task_type="load-git") + r = swh_lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 100 for row in r: assert row["type"] == "load-git" # arguments check args = row["arguments"]["args"] assert len(args) == 0 # kwargs kwargs = row["arguments"]["kwargs"] url = kwargs["url"] assert url.startswith("https://github.com") assert row["policy"] == "recurring" assert row["priority"] is None diff --git a/swh/lister/gitlab/tests/test_lister.py b/swh/lister/gitlab/tests/test_lister.py index 041e969..c22e9bf 100644 --- a/swh/lister/gitlab/tests/test_lister.py +++ b/swh/lister/gitlab/tests/test_lister.py @@ -1,66 +1,72 @@ -# Copyright (C) 2017-2019 The Software Heritage developers +# Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import re import unittest + from datetime import datetime, timedelta +import pytest + from swh.lister.core.tests.test_lister import HttpListerTesterBase from swh.lister.gitlab.lister import GitLabLister logger = logging.getLogger(__name__) class GitLabListerTester(HttpListerTesterBase, unittest.TestCase): Lister = GitLabLister test_re = re.compile(r"^.*/projects.*page=(\d+).*") lister_subdir = "gitlab" good_api_response_file = "data/gitlab.com/api_response.json" bad_api_response_file = "data/gitlab.com/api_empty_response.json" first_index = 1 entries_per_page = 10 convert_type = int def response_headers(self, request): headers = {"RateLimit-Remaining": "1"} if self.request_index(request) == self.first_index: headers.update( {"x-next-page": "3",} ) return headers def mock_rate_quota(self, n, request, context): self.rate_limit += 1 context.status_code = 403 context.headers["RateLimit-Remaining"] = "0" one_second = int((datetime.now() + timedelta(seconds=1.5)).timestamp()) context.headers["RateLimit-Reset"] = str(one_second) return '{"error":"dummy"}' -def test_lister_gitlab(swh_listers, requests_mock_datadir): - lister = swh_listers["gitlab"] +@pytest.fixture +def lister_under_test(): + return "gitlab" + - lister.run() +def test_lister_gitlab(swh_lister, requests_mock_datadir): + swh_lister.run() - r = lister.scheduler.search_tasks(task_type="load-git") + r = swh_lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 10 for row in r: assert row["type"] == "load-git" # arguments check args = row["arguments"]["args"] assert len(args) == 0 # kwargs kwargs = row["arguments"]["kwargs"] url = kwargs["url"] assert url.startswith("https://gitlab.com") assert row["policy"] == "recurring" assert row["priority"] is None diff --git a/swh/lister/gnu/tests/test_lister.py b/swh/lister/gnu/tests/test_lister.py index 92a9bc4..68c0bc5 100644 --- a/swh/lister/gnu/tests/test_lister.py +++ b/swh/lister/gnu/tests/test_lister.py @@ -1,50 +1,56 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging +import pytest + logger = logging.getLogger(__name__) -def test_gnu_lister(swh_listers, requests_mock_datadir): - lister = swh_listers["gnu"] +@pytest.fixture +def lister_under_test(): + return "gnu" + - lister.run() +@pytest.fixture +def test_gnu_lister(swh_lister, requests_mock_datadir): + swh_lister.run() - r = lister.scheduler.search_tasks(task_type="load-archive-files") + r = swh_lister.scheduler.search_tasks(task_type="load-archive-files") assert len(r) == 383 for row in r: assert row["type"] == "load-archive-files" # arguments check args = row["arguments"]["args"] assert len(args) == 0 # kwargs kwargs = row["arguments"]["kwargs"] assert set(kwargs.keys()) == {"url", "artifacts"} url = kwargs["url"] assert url.startswith("https://ftp.gnu.org") url_suffix = url.split("https://ftp.gnu.org")[1] assert "gnu" in url_suffix or "old-gnu" in url_suffix artifacts = kwargs["artifacts"] # check the artifact's structure artifact = artifacts[0] assert set(artifact.keys()) == {"url", "length", "time", "filename", "version"} for artifact in artifacts: logger.debug(artifact) # 'time' is an isoformat string now for key in ["url", "time", "filename", "version"]: assert isinstance(artifact[key], str) assert isinstance(artifact["length"], int) assert row["policy"] == "oneshot" assert row["priority"] is None assert row["retries_left"] == 3 diff --git a/swh/lister/npm/tests/conftest.py b/swh/lister/npm/tests/conftest.py index 507064d..81c3e1c 100644 --- a/swh/lister/npm/tests/conftest.py +++ b/swh/lister/npm/tests/conftest.py @@ -1,23 +1,26 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest @pytest.fixture -def lister_npm(swh_listers): - lister = swh_listers["npm"] +def lister_under_test(): + return "npm" - # Add the load-deb-package in the scheduler backend - lister.scheduler.create_task_type( + +@pytest.fixture +def lister_npm(swh_lister): + # Add the load-npm in the scheduler backend + swh_lister.scheduler.create_task_type( { "type": "load-npm", "description": "Load npm package", "backend_name": "swh.loader.package.tasks.LoadNpm", "default_interval": "1 day", } ) - return lister + return swh_lister diff --git a/swh/lister/packagist/tests/conftest.py b/swh/lister/packagist/tests/conftest.py index 81f8e44..4482346 100644 --- a/swh/lister/packagist/tests/conftest.py +++ b/swh/lister/packagist/tests/conftest.py @@ -1,23 +1,26 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest @pytest.fixture -def lister_packagist(swh_listers): - lister = swh_listers["packagist"] +def lister_under_test(): + return "packagist" + +@pytest.fixture +def lister_packagist(swh_lister): # Amend the scheduler with an unknown yet load-packagist task type - lister.scheduler.create_task_type( + swh_lister.scheduler.create_task_type( { "type": "load-packagist", "description": "Load packagist origin", "backend_name": "swh.loader.package.tasks.LoaderPackagist", "default_interval": "1 day", } ) - return lister + return swh_lister diff --git a/swh/lister/phabricator/tests/conftest.py b/swh/lister/phabricator/tests/conftest.py index 1926122..c4d89d1 100644 --- a/swh/lister/phabricator/tests/conftest.py +++ b/swh/lister/phabricator/tests/conftest.py @@ -1,19 +1,22 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest @pytest.fixture -def lister_phabricator(swh_listers): - lister = swh_listers["phabricator"] +def lister_under_test(): + return "phabricator" + +@pytest.fixture +def lister_phabricator(swh_lister): # Amend the credentials - lister.config = { + swh_lister.config = { "cache_responses": False, - "credentials": {"phabricator": {lister.instance: [{"password": "foo"}]}}, + "credentials": {"phabricator": {swh_lister.instance: [{"password": "foo"}]}}, } - return lister + return swh_lister diff --git a/swh/lister/pypi/tests/conftest.py b/swh/lister/pypi/tests/conftest.py index 5d235f7..528c341 100644 --- a/swh/lister/pypi/tests/conftest.py +++ b/swh/lister/pypi/tests/conftest.py @@ -1,23 +1,26 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest @pytest.fixture -def lister_pypi(swh_listers): - lister = swh_listers["pypi"] +def lister_under_test(): + return "pypi" - # Add the load-deb-package in the scheduler backend - lister.scheduler.create_task_type( + +@pytest.fixture +def lister_pypi(swh_lister): + # Add the load-pypi in the scheduler backend + swh_lister.scheduler.create_task_type( { "type": "load-pypi", "description": "Load PyPI package", "backend_name": "swh.loader.package.tasks.LoadPyPI", "default_interval": "1 day", } ) - return lister + return swh_lister diff --git a/swh/lister/pytest_plugin.py b/swh/lister/pytest_plugin.py index 3d72a74..e153d15 100644 --- a/swh/lister/pytest_plugin.py +++ b/swh/lister/pytest_plugin.py @@ -1,61 +1,43 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import pytest from sqlalchemy import create_engine from swh.lister import get_lister, SUPPORTED_LISTERS from swh.lister.core.models import initialize logger = logging.getLogger(__name__) @pytest.fixture def lister_db_url(postgresql_proc, postgresql): db_url = "postgresql://{user}@{host}:{port}/{dbname}".format( host=postgresql_proc.host, port=postgresql_proc.port, user="postgres", dbname="tests", ) logger.debug("lister db_url: %s", db_url) return db_url @pytest.fixture -def listers_to_instantiate(): - """Fixture to define what listers to instantiate. Because some need dedicated setup. - - """ - return set(SUPPORTED_LISTERS) - {"launchpad"} +def lister_under_test(): + """Fixture to determine which lister to test""" + return "core" @pytest.fixture -def swh_listers( - mock_get_scheduler, lister_db_url, swh_scheduler, listers_to_instantiate -): - listers = {} - - # Prepare schema for all listers - for lister_name in listers_to_instantiate: - lister = get_lister(lister_name, db_url=lister_db_url) - listers[lister_name] = lister +def swh_lister(mock_get_scheduler, lister_db_url, swh_scheduler, lister_under_test): + assert lister_under_test in SUPPORTED_LISTERS + lister = get_lister(lister_under_test, db_url=lister_db_url) initialize(create_engine(lister_db_url), drop_tables=True) - # Add the load-archive-files expected by some listers (gnu, cran, ...) - swh_scheduler.create_task_type( - { - "type": "load-archive-files", - "description": "Load archive files.", - "backend_name": "swh.loader.package.tasks.LoadArchive", - "default_interval": "1 day", - } - ) - - return listers + return lister diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py index 95c19b2..302c3aa 100644 --- a/swh/lister/tests/test_cli.py +++ b/swh/lister/tests/test_cli.py @@ -1,66 +1,69 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from swh.lister.core.lister_base import ListerBase -from swh.lister.cli import get_lister +from swh.lister.cli import get_lister, SUPPORTED_LISTERS from .test_utils import init_db def test_get_lister_wrong_input(): """Unsupported lister should raise""" with pytest.raises(ValueError) as e: get_lister("unknown", "db-url") assert "Invalid lister" in str(e.value) -def test_get_lister(mock_get_scheduler, listers_to_instantiate): +def test_get_lister(mock_get_scheduler): """Instantiating a supported lister should be ok """ db_url = init_db().url() + # exclude listers because they need special instantiation treatment unrelated to + # this test (launchpad: network mock, gnu: scheduler load task) + listers_to_instantiate = set(SUPPORTED_LISTERS) - {"launchpad", "gnu"} for lister_name in listers_to_instantiate: lst = get_lister(lister_name, db_url) assert isinstance(lst, ListerBase) def test_get_lister_override(): """Overriding the lister configuration should populate its config """ db_url = init_db().url() listers = { "gitlab": "https://other.gitlab.uni/api/v4/", "phabricator": "https://somewhere.org/api/diffusion.repository.search", "cgit": "https://some.where/cgit", } # check the override ends up defined in the lister for lister_name, url in listers.items(): lst = get_lister( lister_name, db_url, **{"url": url, "priority": "high", "policy": "oneshot",}, ) assert lst.url == url assert lst.config["priority"] == "high" assert lst.config["policy"] == "oneshot" # check the default urls are used and not the override (since it's not # passed) for lister_name, url in listers.items(): lst = get_lister(lister_name, db_url) # no override so this does not end up in lister's configuration assert "url" not in lst.config assert "priority" not in lst.config assert "oneshot" not in lst.config assert lst.url == lst.DEFAULT_URL