diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py --- a/swh/lister/bitbucket/tests/test_lister.py +++ b/swh/lister/bitbucket/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2019 The Software Heritage developers +# Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -10,12 +10,18 @@ from urllib.parse import unquote import iso8601 +import pytest import requests_mock from swh.lister.bitbucket.lister import BitBucketLister from swh.lister.core.tests.test_lister import HttpListerTester +@pytest.fixture +def lister_under_test(): + return "bitbucket" + + def _convert_type(req_index): """Convert the req_index to its right type according to the model's "indexable" column. @@ -80,11 +86,11 @@ ) -def test_lister_bitbucket(swh_listers, requests_mock_datadir): +def test_lister_bitbucket(swh_lister, requests_mock_datadir): """Simple bitbucket listing should create scheduled tasks (git, hg) """ - lister = swh_listers["bitbucket"] + lister = swh_lister lister.run() diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py --- a/swh/lister/cgit/tests/test_lister.py +++ b/swh/lister/cgit/tests/test_lister.py @@ -1,13 +1,19 @@ -# Copyright (C) 2019 the Software Heritage developers +# Copyright (C) 2019-2020 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import pytest from swh.lister import __version__ -def test_lister_no_page(requests_mock_datadir, swh_listers): - lister = swh_listers["cgit"] +@pytest.fixture +def lister_under_test(): + return "cgit" + + +def test_lister_no_page(requests_mock_datadir, swh_lister): + lister = swh_lister assert lister.url == "https://git.savannah.gnu.org/cgit/" @@ -21,8 +27,8 @@ assert repos[-2] == "http://example.org/cgit/xstarcastle.git/" -def test_lister_model(requests_mock_datadir, swh_listers): - lister = swh_listers["cgit"] +def test_lister_model(requests_mock_datadir, swh_lister): + lister = swh_lister repo = next(lister.get_repos()) @@ -36,8 +42,8 @@ } -def test_lister_with_pages(requests_mock_datadir, swh_listers): - lister = swh_listers["cgit"] +def test_lister_with_pages(requests_mock_datadir, swh_lister): + lister = swh_lister lister.url = "https://git.tizen/cgit/" repos = list(lister.get_repos()) @@ -45,8 +51,8 @@ assert len(repos) == 16 -def test_lister_run(requests_mock_datadir, swh_listers): - lister = swh_listers["cgit"] +def test_lister_run(requests_mock_datadir, swh_lister): + lister = swh_lister lister.url = "https://git.tizen/cgit/" lister.run() @@ -69,8 +75,8 @@ assert row["priority"] is None -def test_lister_requests(requests_mock_datadir, swh_listers): - lister = swh_listers["cgit"] +def test_lister_requests(requests_mock_datadir, swh_lister): + lister = swh_lister lister.url = "https://git.tizen/cgit/" lister.run() diff --git a/swh/lister/cran/tests/conftest.py b/swh/lister/cran/tests/conftest.py --- a/swh/lister/cran/tests/conftest.py +++ b/swh/lister/cran/tests/conftest.py @@ -7,11 +7,14 @@ @pytest.fixture -def lister_cran(swh_listers): - lister = swh_listers["cran"] +def lister_under_test(): + return "cran" - # Add the load-deb-package in the scheduler backend - lister.scheduler.create_task_type( + +@pytest.fixture +def lister_cran(swh_lister): + # Add the load-cran in the scheduler backend + swh_lister.scheduler.create_task_type( { "type": "load-cran", "description": "Load a CRAN package", @@ -20,4 +23,4 @@ } ) - return lister + return swh_lister diff --git a/swh/lister/debian/tests/conftest.py b/swh/lister/debian/tests/conftest.py --- a/swh/lister/debian/tests/conftest.py +++ b/swh/lister/debian/tests/conftest.py @@ -14,14 +14,19 @@ @pytest.fixture -def lister_debian(swh_listers): - lister = swh_listers["debian"] +def lister_under_test(): + return "debian" + +@pytest.fixture +def lister_debian(swh_lister): # Initialize the debian data model - debian_init(lister.db_engine, suites=["stretch"], components=["main", "contrib"]) + debian_init( + swh_lister.db_engine, suites=["stretch"], components=["main", "contrib"] + ) # Add the load-deb-package in the scheduler backend - lister.scheduler.create_task_type( + swh_lister.scheduler.create_task_type( { "type": "load-deb-package", "description": "Load a Debian package", @@ -30,7 +35,7 @@ } ) - return lister + return swh_lister @pytest.fixture diff --git a/swh/lister/gitea/tests/test_lister.py b/swh/lister/gitea/tests/test_lister.py --- a/swh/lister/gitea/tests/test_lister.py +++ b/swh/lister/gitea/tests/test_lister.py @@ -7,6 +7,8 @@ import re import unittest +import pytest + from swh.lister.core.tests.test_lister import HttpListerTesterBase from swh.lister.gitea.lister import GiteaLister @@ -38,8 +40,13 @@ return headers -def test_lister_gitea(swh_listers, requests_mock_datadir): - lister: GiteaLister = swh_listers["gitea"] +@pytest.fixture +def lister_under_test(): + return "gitea" + + +def test_lister_gitea(swh_lister, requests_mock_datadir): + lister: GiteaLister = swh_lister lister.run() r = lister.scheduler.search_tasks(task_type="load-git") diff --git a/swh/lister/github/tests/test_lister.py b/swh/lister/github/tests/test_lister.py --- a/swh/lister/github/tests/test_lister.py +++ b/swh/lister/github/tests/test_lister.py @@ -6,6 +6,7 @@ import re import unittest +import pytest import requests_mock from swh.lister.core.tests.test_lister import HttpListerTester @@ -57,15 +58,18 @@ ) -def test_lister_github(swh_listers, requests_mock_datadir): +@pytest.fixture +def lister_under_test(): + return "github" + + +def test_lister_github(swh_lister, requests_mock_datadir): """Simple github listing should create scheduled tasks """ - lister = swh_listers["github"] - - lister.run() + swh_lister.run() - r = lister.scheduler.search_tasks(task_type="load-git") + r = swh_lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 100 for row in r: diff --git a/swh/lister/gitlab/tests/test_lister.py b/swh/lister/gitlab/tests/test_lister.py --- a/swh/lister/gitlab/tests/test_lister.py +++ b/swh/lister/gitlab/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2019 The Software Heritage developers +# Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -6,8 +6,11 @@ import logging import re import unittest + from datetime import datetime, timedelta +import pytest + from swh.lister.core.tests.test_lister import HttpListerTesterBase from swh.lister.gitlab.lister import GitLabLister @@ -43,12 +46,15 @@ return '{"error":"dummy"}' -def test_lister_gitlab(swh_listers, requests_mock_datadir): - lister = swh_listers["gitlab"] +@pytest.fixture +def lister_under_test(): + return "gitlab" + - lister.run() +def test_lister_gitlab(swh_lister, requests_mock_datadir): + swh_lister.run() - r = lister.scheduler.search_tasks(task_type="load-git") + r = swh_lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 10 for row in r: diff --git a/swh/lister/gnu/tests/test_lister.py b/swh/lister/gnu/tests/test_lister.py --- a/swh/lister/gnu/tests/test_lister.py +++ b/swh/lister/gnu/tests/test_lister.py @@ -1,20 +1,26 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging +import pytest + logger = logging.getLogger(__name__) -def test_gnu_lister(swh_listers, requests_mock_datadir): - lister = swh_listers["gnu"] +@pytest.fixture +def lister_under_test(): + return "gnu" + - lister.run() +@pytest.fixture +def test_gnu_lister(swh_lister, requests_mock_datadir): + swh_lister.run() - r = lister.scheduler.search_tasks(task_type="load-archive-files") + r = swh_lister.scheduler.search_tasks(task_type="load-archive-files") assert len(r) == 383 for row in r: diff --git a/swh/lister/npm/tests/conftest.py b/swh/lister/npm/tests/conftest.py --- a/swh/lister/npm/tests/conftest.py +++ b/swh/lister/npm/tests/conftest.py @@ -7,11 +7,14 @@ @pytest.fixture -def lister_npm(swh_listers): - lister = swh_listers["npm"] +def lister_under_test(): + return "npm" - # Add the load-deb-package in the scheduler backend - lister.scheduler.create_task_type( + +@pytest.fixture +def lister_npm(swh_lister): + # Add the load-npm in the scheduler backend + swh_lister.scheduler.create_task_type( { "type": "load-npm", "description": "Load npm package", @@ -20,4 +23,4 @@ } ) - return lister + return swh_lister diff --git a/swh/lister/packagist/tests/conftest.py b/swh/lister/packagist/tests/conftest.py --- a/swh/lister/packagist/tests/conftest.py +++ b/swh/lister/packagist/tests/conftest.py @@ -7,11 +7,14 @@ @pytest.fixture -def lister_packagist(swh_listers): - lister = swh_listers["packagist"] +def lister_under_test(): + return "packagist" + +@pytest.fixture +def lister_packagist(swh_lister): # Amend the scheduler with an unknown yet load-packagist task type - lister.scheduler.create_task_type( + swh_lister.scheduler.create_task_type( { "type": "load-packagist", "description": "Load packagist origin", @@ -20,4 +23,4 @@ } ) - return lister + return swh_lister diff --git a/swh/lister/phabricator/tests/conftest.py b/swh/lister/phabricator/tests/conftest.py --- a/swh/lister/phabricator/tests/conftest.py +++ b/swh/lister/phabricator/tests/conftest.py @@ -7,13 +7,16 @@ @pytest.fixture -def lister_phabricator(swh_listers): - lister = swh_listers["phabricator"] +def lister_under_test(): + return "phabricator" + +@pytest.fixture +def lister_phabricator(swh_lister): # Amend the credentials - lister.config = { + swh_lister.config = { "cache_responses": False, - "credentials": {"phabricator": {lister.instance: [{"password": "foo"}]}}, + "credentials": {"phabricator": {swh_lister.instance: [{"password": "foo"}]}}, } - return lister + return swh_lister diff --git a/swh/lister/pypi/tests/conftest.py b/swh/lister/pypi/tests/conftest.py --- a/swh/lister/pypi/tests/conftest.py +++ b/swh/lister/pypi/tests/conftest.py @@ -7,11 +7,14 @@ @pytest.fixture -def lister_pypi(swh_listers): - lister = swh_listers["pypi"] +def lister_under_test(): + return "pypi" - # Add the load-deb-package in the scheduler backend - lister.scheduler.create_task_type( + +@pytest.fixture +def lister_pypi(swh_lister): + # Add the load-pypi in the scheduler backend + swh_lister.scheduler.create_task_type( { "type": "load-pypi", "description": "Load PyPI package", @@ -20,4 +23,4 @@ } ) - return lister + return swh_lister diff --git a/swh/lister/pytest_plugin.py b/swh/lister/pytest_plugin.py --- a/swh/lister/pytest_plugin.py +++ b/swh/lister/pytest_plugin.py @@ -29,33 +29,15 @@ @pytest.fixture -def listers_to_instantiate(): - """Fixture to define what listers to instantiate. Because some need dedicated setup. - - """ - return set(SUPPORTED_LISTERS) - {"launchpad"} +def lister_under_test(): + """Fixture to determine which lister to test""" + return "core" @pytest.fixture -def swh_listers( - mock_get_scheduler, lister_db_url, swh_scheduler, listers_to_instantiate -): - listers = {} - - # Prepare schema for all listers - for lister_name in listers_to_instantiate: - lister = get_lister(lister_name, db_url=lister_db_url) - listers[lister_name] = lister +def swh_lister(mock_get_scheduler, lister_db_url, swh_scheduler, lister_under_test): + assert lister_under_test in SUPPORTED_LISTERS + lister = get_lister(lister_under_test, db_url=lister_db_url) initialize(create_engine(lister_db_url), drop_tables=True) - # Add the load-archive-files expected by some listers (gnu, cran, ...) - swh_scheduler.create_task_type( - { - "type": "load-archive-files", - "description": "Load archive files.", - "backend_name": "swh.loader.package.tasks.LoadArchive", - "default_interval": "1 day", - } - ) - - return listers + return lister diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py --- a/swh/lister/tests/test_cli.py +++ b/swh/lister/tests/test_cli.py @@ -7,7 +7,7 @@ from swh.lister.core.lister_base import ListerBase -from swh.lister.cli import get_lister +from swh.lister.cli import get_lister, SUPPORTED_LISTERS from .test_utils import init_db @@ -20,11 +20,14 @@ assert "Invalid lister" in str(e.value) -def test_get_lister(mock_get_scheduler, listers_to_instantiate): +def test_get_lister(mock_get_scheduler): """Instantiating a supported lister should be ok """ db_url = init_db().url() + # exclude listers because they need special instantiation treatment unrelated to + # this test (launchpad: network mock, gnu: scheduler load task) + listers_to_instantiate = set(SUPPORTED_LISTERS) - {"launchpad", "gnu"} for lister_name in listers_to_instantiate: lst = get_lister(lister_name, db_url) assert isinstance(lst, ListerBase)