diff --git a/conftest.py b/conftest.py index cd5ed25..01b5b0d 100644 --- a/conftest.py +++ b/conftest.py @@ -1,67 +1,67 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import pytest import yaml from typing import Any, Dict -from swh.scheduler.tests.conftest import swh_app # noqa +pytest_plugins = ["swh.scheduler.pytest_plugin", "swh.storage.pytest_plugin"] @pytest.fixture def swh_loader_config(swh_storage_postgresql) -> Dict[str, Any]: return { "storage": { "cls": "pipeline", "steps": [ {"cls": "retry"}, {"cls": "filter"}, {"cls": "buffer"}, { "cls": "local", "db": swh_storage_postgresql.dsn, "objstorage": {"cls": "memory", "args": {}}, }, ], }, "deposit": { "url": "https://deposit.softwareheritage.org/1/private", "auth": {"username": "user", "password": "pass",}, }, } @pytest.fixture def swh_config(swh_loader_config, monkeypatch, tmp_path): conffile = os.path.join(str(tmp_path), "loader.yml") with open(conffile, "w") as f: f.write(yaml.dump(swh_loader_config)) monkeypatch.setenv("SWH_CONFIG_FILENAME", conffile) return conffile @pytest.fixture(autouse=True, scope="session") def swh_proxy(): """Automatically inject this fixture in all tests to ensure no outside connection takes place. """ os.environ["http_proxy"] = "http://localhost:999" os.environ["https_proxy"] = "http://localhost:999" -@pytest.fixture(scope="session") # type: ignore # expected redefinition -def celery_includes(): - return [ +@pytest.fixture(scope="session") +def swh_scheduler_celery_includes(swh_scheduler_celery_includes): + return swh_scheduler_celery_includes + [ "swh.loader.package.archive.tasks", "swh.loader.package.cran.tasks", "swh.loader.package.debian.tasks", "swh.loader.package.deposit.tasks", "swh.loader.package.npm.tasks", "swh.loader.package.pypi.tasks", "swh.loader.package.nixguix.tasks", ] diff --git a/pytest.ini b/pytest.ini index 1d006ad..cd76362 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,9 @@ [pytest] +# Drop this when swh.scheduler stops exporting this plugin automatically +addopts = -p no:pytest_swh_scheduler -p no:pytest_swh_storage + norecursedirs = docs markers = db: marks tests as using a db (deselect with '-m "not db"') fs: marks tests as using the filesystem (deselect with '-m "not fs"') diff --git a/requirements-test.txt b/requirements-test.txt index aab3d89..0532160 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,6 @@ pytest pytest-mock requests_mock swh-core[testing] -swh-scheduler[testing] +swh-scheduler[testing] >= 0.5.0 swh-storage[testing] diff --git a/swh/loader/package/archive/tests/test_tasks.py b/swh/loader/package/archive/tests/test_tasks.py index 09fc47a..2edf325 100644 --- a/swh/loader/package/archive/tests/test_tasks.py +++ b/swh/loader/package/archive/tests/test_tasks.py @@ -1,19 +1,21 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -def test_archive_loader(mocker, swh_app, celery_session_worker, swh_config): +def test_archive_loader( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): mock_loader = mocker.patch("swh.loader.package.archive.loader.ArchiveLoader.load") mock_loader.return_value = {"status": "eventful"} - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.loader.package.archive.tasks.LoadArchive", kwargs={"url": "some-url", "artifacts": []}, ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/cran/tests/test_tasks.py b/swh/loader/package/cran/tests/test_tasks.py index aa4ea00..b54c729 100644 --- a/swh/loader/package/cran/tests/test_tasks.py +++ b/swh/loader/package/cran/tests/test_tasks.py @@ -1,22 +1,24 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -def test_cran_loader(mocker, swh_app, celery_session_worker, swh_config): +def test_cran_loader( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): mock_loader = mocker.patch("swh.loader.package.cran.loader.CRANLoader.load") mock_loader.return_value = {"status": "eventful"} - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.loader.package.cran.tasks.LoadCRAN", kwargs={ "url": "some-url", "artifacts": {"version": "1.2.3", "url": "artifact-url"}, }, ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/debian/tests/test_tasks.py b/swh/loader/package/debian/tests/test_tasks.py index 1cc1c9d..62b39af 100644 --- a/swh/loader/package/debian/tests/test_tasks.py +++ b/swh/loader/package/debian/tests/test_tasks.py @@ -1,19 +1,21 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -def test_debian_loader(mocker, swh_app, celery_session_worker, swh_config): +def test_debian_loader( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): mock_loader = mocker.patch("swh.loader.package.debian.loader.DebianLoader.load") mock_loader.return_value = {"status": "eventful"} - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.loader.package.debian.tasks.LoadDebian", kwargs={"url": "some-url", "date": "some-date", "packages": {}}, ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/deposit/tests/test_tasks.py b/swh/loader/package/deposit/tests/test_tasks.py index dc7aa7c..0e2b739 100644 --- a/swh/loader/package/deposit/tests/test_tasks.py +++ b/swh/loader/package/deposit/tests/test_tasks.py @@ -1,19 +1,21 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -def test_deposit_loader(mocker, swh_app, celery_session_worker, swh_config): +def test_deposit_loader( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): mock_loader = mocker.patch("swh.loader.package.deposit.loader.DepositLoader.load") mock_loader.return_value = {"status": "eventful"} - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.loader.package.deposit.tasks.LoadDeposit", kwargs={"url": "some-url", "deposit_id": "some-d-id",}, ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/nixguix/tests/test_tasks.py b/swh/loader/package/nixguix/tests/test_tasks.py index df66239..8b765a6 100644 --- a/swh/loader/package/nixguix/tests/test_tasks.py +++ b/swh/loader/package/nixguix/tests/test_tasks.py @@ -1,27 +1,29 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -def test_nixguix_loader(mocker, swh_app, celery_session_worker, swh_config): +def test_nixguix_loader( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): mock_loader = mocker.patch("swh.loader.package.nixguix.loader.NixGuixLoader.load") mock_loader.return_value = {"status": "eventful"} mock_retrieve_sources = mocker.patch( "swh.loader.package.nixguix.loader.retrieve_sources" ) mock_retrieve_sources.return_value = { "version": 1, "sources": [], "revision": "some-revision", } - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.loader.package.nixguix.tasks.LoadNixguix", kwargs=dict(url="some-url") ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/npm/tests/test_tasks.py b/swh/loader/package/npm/tests/test_tasks.py index 9501907..c0b3a5f 100644 --- a/swh/loader/package/npm/tests/test_tasks.py +++ b/swh/loader/package/npm/tests/test_tasks.py @@ -1,19 +1,21 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -def test_npm_loader(mocker, swh_app, celery_session_worker, swh_config): +def test_npm_loader( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): mock_loader = mocker.patch("swh.loader.package.npm.loader.NpmLoader.load") mock_loader.return_value = {"status": "eventful"} - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.loader.package.npm.tasks.LoadNpm", kwargs=dict(url="https://www.npmjs.com/package/some-package"), ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/pypi/tests/test_tasks.py b/swh/loader/package/pypi/tests/test_tasks.py index 152fd45..19886b5 100644 --- a/swh/loader/package/pypi/tests/test_tasks.py +++ b/swh/loader/package/pypi/tests/test_tasks.py @@ -1,18 +1,20 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -def test_pypi_loader(mocker, swh_app, celery_session_worker, swh_config): +def test_pypi_loader( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): mock_loader = mocker.patch("swh.loader.package.pypi.loader.PyPILoader.load") mock_loader.return_value = {"status": "eventful"} - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.loader.package.pypi.tasks.LoadPyPI", kwargs=dict(url="some-url") ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} diff --git a/tox.ini b/tox.ini index e5faf28..225614c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,37 +1,38 @@ [tox] envlist=black,flake8,mypy,py3 [testenv] extras = testing deps = swh.core[http] - swh.storage[testing] # otherwise, somehow pytest-postgresql is not seen + swh.storage[testing] + swh.scheduler[testing] >= 0.5.0 pytest-cov dev: pdbpp commands = pytest \ !dev: --cov={envsitepackagesdir}/swh/loader/ --cov-branch \ {envsitepackagesdir}/swh/loader/ {posargs} [testenv:black] skip_install = true deps = black commands = {envpython} -m black --check swh [testenv:flake8] skip_install = true deps = flake8 commands = {envpython} -m flake8 [testenv:mypy] extras = testing deps = mypy commands = mypy swh