diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py --- a/swh/lister/bitbucket/tests/test_lister.py +++ b/swh/lister/bitbucket/tests/test_lister.py @@ -17,11 +17,6 @@ from swh.lister.core.tests.test_lister import HttpListerTester -@pytest.fixture -def lister_under_test(): - return "bitbucket" - - def _convert_type(req_index): """Convert the req_index to its right type according to the model's "indexable" column. @@ -86,15 +81,18 @@ ) -def test_lister_bitbucket(swh_lister, requests_mock_datadir): +@pytest.fixture +def lister_under_test(): + return "bitbucket" + + +def test_lister_bitbucket(initialized_lister, requests_mock_datadir): """Simple bitbucket listing should create scheduled tasks (git, hg) """ - lister = swh_lister - - lister.run() + initialized_lister.run() - r = lister.scheduler.search_tasks(task_type="load-hg") + r = initialized_lister.scheduler.search_tasks(task_type="load-hg") assert len(r) == 9 for row in r: @@ -110,7 +108,7 @@ assert row["policy"] == "recurring" assert row["priority"] is None - r = lister.scheduler.search_tasks(task_type="load-git") + r = initialized_lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 1 for row in r: diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py --- a/swh/lister/cgit/tests/test_lister.py +++ b/swh/lister/cgit/tests/test_lister.py @@ -12,12 +12,10 @@ return "cgit" -def test_lister_no_page(requests_mock_datadir, swh_lister): - lister = swh_lister +def test_lister_no_page(requests_mock_datadir, initialized_lister): + assert initialized_lister.url == "https://git.savannah.gnu.org/cgit/" - assert lister.url == "https://git.savannah.gnu.org/cgit/" - - repos = list(lister.get_repos()) + repos = list(initialized_lister.get_repos()) assert len(repos) == 977 assert repos[0] == "https://git.savannah.gnu.org/cgit/elisp-es.git/" @@ -27,12 +25,10 @@ assert repos[-2] == "http://example.org/cgit/xstarcastle.git/" -def test_lister_model(requests_mock_datadir, swh_lister): - lister = swh_lister - - repo = next(lister.get_repos()) +def test_lister_model(requests_mock_datadir, initialized_lister): + repo = next(initialized_lister.get_repos()) - model = lister.build_model(repo) + model = initialized_lister.build_model(repo) assert model == { "uid": "https://git.savannah.gnu.org/cgit/elisp-es.git/", "name": "elisp-es.git", @@ -42,21 +38,19 @@ } -def test_lister_with_pages(requests_mock_datadir, swh_lister): - lister = swh_lister - lister.url = "https://git.tizen/cgit/" +def test_lister_with_pages(requests_mock_datadir, initialized_lister): + initialized_lister.url = "https://git.tizen/cgit/" - repos = list(lister.get_repos()) + repos = list(initialized_lister.get_repos()) # we should have 16 repos (listed on 3 pages) assert len(repos) == 16 -def test_lister_run(requests_mock_datadir, swh_lister): - lister = swh_lister - lister.url = "https://git.tizen/cgit/" - lister.run() +def test_lister_run(requests_mock_datadir, initialized_lister): + initialized_lister.url = "https://git.tizen/cgit/" + initialized_lister.run() - r = lister.scheduler.search_tasks(task_type="load-git") + r = initialized_lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 16 for row in r: @@ -75,10 +69,9 @@ assert row["priority"] is None -def test_lister_requests(requests_mock_datadir, swh_lister): - lister = swh_lister - lister.url = "https://git.tizen/cgit/" - lister.run() +def test_lister_requests(requests_mock_datadir, initialized_lister): + initialized_lister.url = "https://git.tizen/cgit/" + initialized_lister.run() assert len(requests_mock_datadir.request_history) != 0 for request in requests_mock_datadir.request_history: diff --git a/swh/lister/cran/tests/conftest.py b/swh/lister/cran/tests/conftest.py --- a/swh/lister/cran/tests/conftest.py +++ b/swh/lister/cran/tests/conftest.py @@ -12,7 +12,7 @@ @pytest.fixture -def lister_cran(swh_lister): +def initialized_lister(swh_lister): # Add the load-cran in the scheduler backend swh_lister.scheduler.create_task_type( { diff --git a/swh/lister/cran/tests/test_lister.py b/swh/lister/cran/tests/test_lister.py --- a/swh/lister/cran/tests/test_lister.py +++ b/swh/lister/cran/tests/test_lister.py @@ -28,18 +28,16 @@ @patch("swh.lister.cran.lister.read_cran_data") -def test_cran_lister_cran(mock_cran, datadir, lister_cran): - lister = lister_cran - +def test_cran_lister_cran(mock_cran, datadir, initialized_lister): with open(path.join(datadir, "list-r-packages.json")) as f: data = json.loads(f.read()) mock_cran.return_value = data assert len(data) == 6 - lister.run() + initialized_lister.run() - r = lister.scheduler.search_tasks(task_type="load-cran") + r = initialized_lister.scheduler.search_tasks(task_type="load-cran") assert len(r) == 6 for row in r: @@ -63,7 +61,7 @@ origin_url = kwargs["url"] record = ( - lister.db_session.query(lister.MODEL) + initialized_lister.db_session.query(initialized_lister.MODEL) .filter(origin_url == origin_url) .first() ) diff --git a/swh/lister/debian/tests/conftest.py b/swh/lister/debian/tests/conftest.py --- a/swh/lister/debian/tests/conftest.py +++ b/swh/lister/debian/tests/conftest.py @@ -19,7 +19,7 @@ @pytest.fixture -def lister_debian(swh_lister): +def initialized_lister(swh_lister): # Initialize the debian data model debian_init( swh_lister.db_engine, suites=["stretch"], components=["main", "contrib"] diff --git a/swh/lister/debian/tests/test_lister.py b/swh/lister/debian/tests/test_lister.py --- a/swh/lister/debian/tests/test_lister.py +++ b/swh/lister/debian/tests/test_lister.py @@ -9,14 +9,14 @@ logger = logging.getLogger(__name__) -def test_lister_debian(lister_debian, datadir, requests_mock_datadir): +def test_lister_debian(initialized_lister, datadir, requests_mock_datadir): """Simple debian listing should create scheduled tasks """ # Run the lister - lister_debian.run() + initialized_lister.run() - r = lister_debian.scheduler.search_tasks(task_type="load-deb-package") + r = initialized_lister.scheduler.search_tasks(task_type="load-deb-package") assert len(r) == 151 for row in r: diff --git a/swh/lister/gitea/tests/test_lister.py b/swh/lister/gitea/tests/test_lister.py --- a/swh/lister/gitea/tests/test_lister.py +++ b/swh/lister/gitea/tests/test_lister.py @@ -45,11 +45,9 @@ return "gitea" -def test_lister_gitea(swh_lister, requests_mock_datadir): - lister: GiteaLister = swh_lister - - lister.run() - r = lister.scheduler.search_tasks(task_type="load-git") +def test_lister_gitea(initialized_lister, requests_mock_datadir): + initialized_lister.run() + r = initialized_lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 3 for row in r: diff --git a/swh/lister/github/tests/test_lister.py b/swh/lister/github/tests/test_lister.py --- a/swh/lister/github/tests/test_lister.py +++ b/swh/lister/github/tests/test_lister.py @@ -63,13 +63,13 @@ return "github" -def test_lister_github(swh_lister, requests_mock_datadir): +def test_lister_github(initialized_lister, requests_mock_datadir): """Simple github listing should create scheduled tasks """ - swh_lister.run() + initialized_lister.run() - r = swh_lister.scheduler.search_tasks(task_type="load-git") + r = initialized_lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 100 for row in r: diff --git a/swh/lister/gitlab/tests/test_lister.py b/swh/lister/gitlab/tests/test_lister.py --- a/swh/lister/gitlab/tests/test_lister.py +++ b/swh/lister/gitlab/tests/test_lister.py @@ -51,10 +51,10 @@ return "gitlab" -def test_lister_gitlab(swh_lister, requests_mock_datadir): - swh_lister.run() +def test_lister_gitlab(initialized_lister, requests_mock_datadir): + initialized_lister.run() - r = swh_lister.scheduler.search_tasks(task_type="load-git") + r = initialized_lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 10 for row in r: diff --git a/swh/lister/gnu/tests/test_lister.py b/swh/lister/gnu/tests/test_lister.py --- a/swh/lister/gnu/tests/test_lister.py +++ b/swh/lister/gnu/tests/test_lister.py @@ -17,10 +17,10 @@ @pytest.fixture -def test_gnu_lister(swh_lister, requests_mock_datadir): - swh_lister.run() +def test_gnu_lister(initialized_lister, requests_mock_datadir): + initialized_lister.run() - r = swh_lister.scheduler.search_tasks(task_type="load-archive-files") + r = initialized_lister.scheduler.search_tasks(task_type="load-archive-files") assert len(r) == 383 for row in r: diff --git a/swh/lister/launchpad/tests/conftest.py b/swh/lister/launchpad/tests/conftest.py --- a/swh/lister/launchpad/tests/conftest.py +++ b/swh/lister/launchpad/tests/conftest.py @@ -14,7 +14,7 @@ @pytest.fixture -def lister_launchpad(datadir, lister_db_url, swh_scheduler): +def initialized_lister(datadir, lister_db_url, swh_scheduler): class Collection: entries = [] diff --git a/swh/lister/launchpad/tests/test_lister.py b/swh/lister/launchpad/tests/test_lister.py --- a/swh/lister/launchpad/tests/test_lister.py +++ b/swh/lister/launchpad/tests/test_lister.py @@ -4,15 +4,16 @@ # See top-level LICENSE file for more information -def test_launchpad_lister(lister_launchpad, datadir): +def test_launchpad_lister(initialized_lister, datadir): - lister_launchpad.run() + initialized_lister.run() assert ( - len(lister_launchpad.launchpad.git_repositories.getRepositories.mock_calls) == 3 + len(initialized_lister.launchpad.git_repositories.getRepositories.mock_calls) + == 3 ) - r = lister_launchpad.scheduler.search_tasks(task_type="load-git") + r = initialized_lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 30 for row in r: diff --git a/swh/lister/npm/tests/conftest.py b/swh/lister/npm/tests/conftest.py --- a/swh/lister/npm/tests/conftest.py +++ b/swh/lister/npm/tests/conftest.py @@ -12,7 +12,7 @@ @pytest.fixture -def lister_npm(swh_lister): +def initialized_lister(swh_lister): # Add the load-npm in the scheduler backend swh_lister.scheduler.create_task_type( { diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py --- a/swh/lister/npm/tests/test_lister.py +++ b/swh/lister/npm/tests/test_lister.py @@ -75,17 +75,17 @@ assert row["priority"] is None -def test_lister_npm_basic_listing(lister_npm, requests_mock_datadir): - lister_npm.run() +def test_lister_npm_basic_listing(initialized_lister, requests_mock_datadir): + initialized_lister.run() - tasks = lister_npm.scheduler.search_tasks(task_type="load-npm") + tasks = initialized_lister.scheduler.search_tasks(task_type="load-npm") assert len(tasks) == 100 check_tasks(tasks) -def test_lister_npm_listing_pagination(lister_npm, requests_mock_datadir): - lister = lister_npm +def test_lister_npm_listing_pagination(initialized_lister, requests_mock_datadir): + lister = initialized_lister # Patch per page pagination lister.per_page = 10 + 1 lister.PATH_TEMPLATE = lister.PATH_TEMPLATE.replace( diff --git a/swh/lister/packagist/tests/conftest.py b/swh/lister/packagist/tests/conftest.py --- a/swh/lister/packagist/tests/conftest.py +++ b/swh/lister/packagist/tests/conftest.py @@ -12,7 +12,7 @@ @pytest.fixture -def lister_packagist(swh_lister): +def initialized_lister(swh_lister): # Amend the scheduler with an unknown yet load-packagist task type swh_lister.scheduler.create_task_type( { diff --git a/swh/lister/packagist/tests/test_lister.py b/swh/lister/packagist/tests/test_lister.py --- a/swh/lister/packagist/tests/test_lister.py +++ b/swh/lister/packagist/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -79,10 +79,10 @@ assert actual_url == expected_url -def test_packagist_lister(lister_packagist, requests_mock_datadir): - lister_packagist.run() +def test_packagist_lister(initialized_lister, requests_mock_datadir): + initialized_lister.run() - r = lister_packagist.scheduler.search_tasks(task_type="load-packagist") + r = initialized_lister.scheduler.search_tasks(task_type="load-packagist") assert len(r) == 5 for row in r: diff --git a/swh/lister/phabricator/tests/conftest.py b/swh/lister/phabricator/tests/conftest.py --- a/swh/lister/phabricator/tests/conftest.py +++ b/swh/lister/phabricator/tests/conftest.py @@ -12,7 +12,7 @@ @pytest.fixture -def lister_phabricator(swh_lister): +def initialized_lister(swh_lister): # Amend the credentials swh_lister.config = { "cache_responses": False, diff --git a/swh/lister/phabricator/tests/test_lister.py b/swh/lister/phabricator/tests/test_lister.py --- a/swh/lister/phabricator/tests/test_lister.py +++ b/swh/lister/phabricator/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -111,13 +111,12 @@ self.assertTrue(task["status"] != "disabled") -def test_phabricator_lister(lister_phabricator, requests_mock_datadir): - lister = lister_phabricator - assert lister.url == lister.DEFAULT_URL - assert lister.instance == "forge.softwareheritage.org" - lister.run() +def test_phabricator_lister(initialized_lister, requests_mock_datadir): + assert initialized_lister.url == initialized_lister.DEFAULT_URL + assert initialized_lister.instance == "forge.softwareheritage.org" + initialized_lister.run() - r = lister.scheduler.search_tasks(task_type="load-git") + r = initialized_lister.scheduler.search_tasks(task_type="load-git") assert len(r) == 10 for row in r: @@ -129,7 +128,7 @@ # kwargs kwargs = row["arguments"]["kwargs"] url = kwargs["url"] - assert lister.instance in url + assert initialized_lister.instance in url assert row["policy"] == "recurring" assert row["priority"] is None diff --git a/swh/lister/pypi/tests/conftest.py b/swh/lister/pypi/tests/conftest.py --- a/swh/lister/pypi/tests/conftest.py +++ b/swh/lister/pypi/tests/conftest.py @@ -12,7 +12,7 @@ @pytest.fixture -def lister_pypi(swh_lister): +def initialized_lister(swh_lister): # Add the load-pypi in the scheduler backend swh_lister.scheduler.create_task_type( { diff --git a/swh/lister/pypi/tests/test_lister.py b/swh/lister/pypi/tests/test_lister.py --- a/swh/lister/pypi/tests/test_lister.py +++ b/swh/lister/pypi/tests/test_lister.py @@ -1,13 +1,13 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -def test_pypi_lister(lister_pypi, requests_mock_datadir): - lister_pypi.run() +def test_pypi_lister(initialized_lister, requests_mock_datadir): + initialized_lister.run() - r = lister_pypi.scheduler.search_tasks(task_type="load-pypi") + r = initialized_lister.scheduler.search_tasks(task_type="load-pypi") assert len(r) == 4 for row in r: diff --git a/swh/lister/pytest_plugin.py b/swh/lister/pytest_plugin.py --- a/swh/lister/pytest_plugin.py +++ b/swh/lister/pytest_plugin.py @@ -36,8 +36,21 @@ @pytest.fixture def swh_lister(mock_get_scheduler, lister_db_url, swh_scheduler, lister_under_test): + """Main lister instance, override `lister_under_test` to declare what lister to + instantiate (e.g. github, gitlab, npm, pypi, etc...). Check the SUPPORTED_LISTERS + variable for further example. + + """ assert lister_under_test in SUPPORTED_LISTERS lister = get_lister(lister_under_test, db_url=lister_db_url) initialize(create_engine(lister_db_url), drop_tables=True) - return lister + + +@pytest.fixture +def initialized_lister(swh_lister): + """Fixture to allow further lister initialization (e.g. add tasks to the scheduler, + etc...) + + """ + return swh_lister