diff --git a/swh/loader/core/tests/test_tasks.py b/swh/loader/core/tests/test_tasks.py index effe27b..5b006c6 100644 --- a/swh/loader/core/tests/test_tasks.py +++ b/swh/loader/core/tests/test_tasks.py @@ -1,53 +1,42 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid import pytest from swh.scheduler.model import ListedOrigin, Lister -from swh.scheduler.utils import create_origin_task_dict NAMESPACE = "swh.loader.core" @pytest.fixture def nixguix_lister(): return Lister(name="nixguix", instance_name="example", id=uuid.uuid4()) @pytest.mark.parametrize("loader_name", ["Content", "Directory"]) -def test_tasks_loader_for_listed_origin( - mocker, - swh_scheduler_celery_app, - swh_scheduler_celery_worker, - swh_config, +def test_loader_tasks_for_listed_origin( + loading_task_creation_for_listed_origin_test, nixguix_lister, loader_name, ): - mock_load = mocker.patch(f"{NAMESPACE}.loader.{loader_name}Loader.load") - mock_load.return_value = {"status": "eventful"} listed_origin = ListedOrigin( lister_id=nixguix_lister.id, url="https://example.org/artifact/artifact", visit_type=loader_name.lower(), extra_loader_arguments={ "fallback_urls": ["https://example.org/mirror/artifact-0.0.1.pkg.xz"], "checksums": {"sha256": "some-valid-checksum"}, }, ) - task_dict = create_origin_task_dict(listed_origin, nixguix_lister) - - res = swh_scheduler_celery_app.send_task( - f"{NAMESPACE}.tasks.Load{loader_name}", - kwargs=task_dict["arguments"]["kwargs"], + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.{loader_name}Loader", + task_function_name=f"{NAMESPACE}.tasks.Load{loader_name}", + lister=nixguix_lister, + listed_origin=listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/arch/tests/test_tasks.py b/swh/loader/package/arch/tests/test_tasks.py index b5178ac..13f5768 100644 --- a/swh/loader/package/arch/tests/test_tasks.py +++ b/swh/loader/package/arch/tests/test_tasks.py @@ -1,40 +1,59 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid -def test_tasks_arch_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config -): - mock_load = mocker.patch("swh.loader.package.arch.loader.ArchLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.arch.tasks.LoadArch", - kwargs=dict( - url="some-url/packages/s/some-package", - artifacts=[ +import pytest + +from swh.scheduler.model import ListedOrigin, Lister + +NAMESPACE = "swh.loader.package.arch" + + +@pytest.fixture +def arch_lister(): + return Lister(name="arch", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def arch_listed_origin(arch_lister): + return ListedOrigin( + lister_id=arch_lister.id, + url="some-url/packages/s/some-package", + visit_type="arch", + extra_loader_arguments={ + "artifacts": [ { "version": "0.0.1", "url": "https://somewhere/some-package-0.0.1.pkg.xz", "filename": "some-package-0.0.1.pkg.xz", "length": 42, } ], - arch_metadata=[ + "arch_metadata": [ { "version": "0.0.1", "arch": "aarch64", "name": "some-package", "repo": "community", "last_modified": "1970-01-01T21:08:14", } ], - ), + }, + ) + + +def test_arch_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, + arch_lister, + arch_listed_origin, +): + + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.ArchLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadArch", + lister=arch_lister, + listed_origin=arch_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/archive/tests/test_tasks.py b/swh/loader/package/archive/tests/test_tasks.py index 0d2448f..dc56908 100644 --- a/swh/loader/package/archive/tests/test_tasks.py +++ b/swh/loader/package/archive/tests/test_tasks.py @@ -1,92 +1,52 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid import pytest from swh.scheduler.model import ListedOrigin, Lister -from swh.scheduler.utils import create_origin_task_dict - -@pytest.fixture(autouse=True) -def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): - pass +NAMESPACE = "swh.loader.package.archive" @pytest.fixture def archive_lister(): return Lister(name="archive-lister", instance_name="example", id=uuid.uuid4()) @pytest.fixture def archive_listed_origin(archive_lister): return ListedOrigin( lister_id=archive_lister.id, url="https://example.org/archives", visit_type="tar", extra_loader_arguments={ - "artifacts": [], + "artifacts": [ + { + "time": "2010-08-14T01:41:56", + "url": "https://example.org/archives/project-v1.0.0.tar.gz", + "filename": "project-v1.0.0.tar.gz", + "version": "1.0.0", + "length": 2500, + } + ], "snapshot_append": True, }, ) -def test_tasks_archive_loader( - mocker, - swh_scheduler_celery_app, -): - mock_load = mocker.patch("swh.loader.package.archive.loader.ArchiveLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.archive.tasks.LoadArchive", - kwargs=dict(url="https://gnu.org/", artifacts=[]), - ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} - - -def test_tasks_archive_loader_snapshot_append( - mocker, - swh_scheduler_celery_app, -): - mock_load = mocker.patch("swh.loader.package.archive.loader.ArchiveLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.archive.tasks.LoadArchive", - kwargs=dict(url="https://gnu.org/", artifacts=[], snapshot_append=True), - ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} - - -def test_tasks_archive_loader_for_listed_origin( - mocker, - swh_scheduler_celery_app, +def test_archive_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, archive_lister, archive_listed_origin, ): - mock_load = mocker.patch("swh.loader.package.archive.loader.ArchiveLoader.load") - mock_load.return_value = {"status": "eventful"} - - task_dict = create_origin_task_dict(archive_listed_origin, archive_lister) - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.archive.tasks.LoadArchive", - kwargs=task_dict["arguments"]["kwargs"], + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.ArchiveLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadArchive", + lister=archive_lister, + listed_origin=archive_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/aur/tests/test_tasks.py b/swh/loader/package/aur/tests/test_tasks.py index b3ebafa..f814dc1 100644 --- a/swh/loader/package/aur/tests/test_tasks.py +++ b/swh/loader/package/aur/tests/test_tasks.py @@ -1,38 +1,57 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid -def test_tasks_aur_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config -): - mock_load = mocker.patch("swh.loader.package.aur.loader.AurLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.aur.tasks.LoadAur", - kwargs=dict( - url="https://somewhere/some-package.git", - artifacts=[ +import pytest + +from swh.scheduler.model import ListedOrigin, Lister + +NAMESPACE = "swh.loader.package.aur" + + +@pytest.fixture +def aur_lister(): + return Lister(name="aur", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def aur_listed_origin(aur_lister): + return ListedOrigin( + lister_id=aur_lister.id, + url="https://somewhere/some-package.git", + visit_type="aur", + extra_loader_arguments={ + "artifacts": [ { "filename": "some-package.tar.gz", "url": "https://somewhere/some-package.tar.gz", "version": "0.0.1", } ], - aur_metadata=[ + "aur_metadata": [ { "version": "0.0.1", "project_url": "https://somewhere/some-package", "last_update": "1970-01-01T21:08:14", "pkgname": "some-package", } ], - ), + }, + ) + + +def test_aur_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, + aur_lister, + aur_listed_origin, +): + + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.AurLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadAur", + lister=aur_lister, + listed_origin=aur_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/cpan/tests/test_tasks.py b/swh/loader/package/cpan/tests/test_tasks.py index d3e52b8..dc8cf1e 100644 --- a/swh/loader/package/cpan/tests/test_tasks.py +++ b/swh/loader/package/cpan/tests/test_tasks.py @@ -1,23 +1,40 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid -def test_tasks_cpan_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +import pytest + +from swh.scheduler.model import ListedOrigin, Lister + +NAMESPACE = "swh.loader.package.cpan" + + +@pytest.fixture +def cpan_lister(): + return Lister(name="cpan", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def cpan_listed_origin(cpan_lister): + return ListedOrigin( + lister_id=cpan_lister.id, + url="https://metacpan.org/dist/Software-Packager", + visit_type="cpan", + ) + + +def test_cpan_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, + cpan_lister, + cpan_listed_origin, ): - mock_load = mocker.patch("swh.loader.package.cpan.loader.CpanLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.cpan.tasks.LoadCpan", - kwargs=dict( - url="some-url/api/packages/some-package", - ), + + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.CpanLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadCpan", + lister=cpan_lister, + listed_origin=cpan_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/cran/tests/test_tasks.py b/swh/loader/package/cran/tests/test_tasks.py index e2b5de2..8ea1487 100644 --- a/swh/loader/package/cran/tests/test_tasks.py +++ b/swh/loader/package/cran/tests/test_tasks.py @@ -1,76 +1,43 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid import pytest from swh.scheduler.model import ListedOrigin, Lister -from swh.scheduler.utils import create_origin_task_dict - -@pytest.fixture(autouse=True) -def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): - pass +NAMESPACE = "swh.loader.package.cran" @pytest.fixture def cran_lister(): - return Lister(name="cran-lister", instance_name="example", id=uuid.uuid4()) + return Lister(name="cran", instance_name="example", id=uuid.uuid4()) @pytest.fixture def cran_listed_origin(cran_lister): return ListedOrigin( lister_id=cran_lister.id, url="https://cran.example.org/project", visit_type="cran", extra_loader_arguments={ "artifacts": [{"version": "1.2.3", "url": "artifact-url"}], }, ) -def test_tasks_cran_loader( - mocker, - swh_scheduler_celery_app, -): - mock_load = mocker.patch("swh.loader.package.cran.loader.CRANLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.cran.tasks.LoadCRAN", - kwargs=dict( - url="some-url", - artifacts=[{"version": "1.2.3", "url": "artifact-url"}], - ), - ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} - - -def test_tasks_cran_loader_for_listed_origin( - mocker, - swh_scheduler_celery_app, +def test_cran_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, cran_lister, cran_listed_origin, ): - mock_load = mocker.patch("swh.loader.package.cran.loader.CRANLoader.load") - mock_load.return_value = {"status": "eventful"} - - task_dict = create_origin_task_dict(cran_listed_origin, cran_lister) - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.cran.tasks.LoadCRAN", - kwargs=task_dict["arguments"]["kwargs"], + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.CRANLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadCRAN", + lister=cran_lister, + listed_origin=cran_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/crates/tests/test_tasks.py b/swh/loader/package/crates/tests/test_tasks.py index b60e2d4..2c1b459 100644 --- a/swh/loader/package/crates/tests/test_tasks.py +++ b/swh/loader/package/crates/tests/test_tasks.py @@ -1,24 +1,43 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid -def test_tasks_crates_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +import pytest + +from swh.scheduler.model import ListedOrigin, Lister + +NAMESPACE = "swh.loader.package.crates" + + +@pytest.fixture +def crates_lister(): + return Lister(name="crates", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def crates_listed_origin(crates_lister): + return ListedOrigin( + lister_id=crates_lister.id, + url="some-url/api/v1/crates/some-package", + visit_type="crates", + extra_loader_arguments={ + "artifacts": [{"version": "0.0.1", "url": "some-package-0.0.1.crate"}], + }, + ) + + +def test_crates_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, + crates_lister, + crates_listed_origin, ): - mock_load = mocker.patch("swh.loader.package.crates.loader.CratesLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.crates.tasks.LoadCrates", - kwargs=dict( - url="some-url/api/v1/crates/some-package", - artifacts=[{"version": "0.0.1", "url": "some-package-0.0.1.crate"}], - ), + + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.CratesLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadCrates", + lister=crates_lister, + listed_origin=crates_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/debian/tests/test_tasks.py b/swh/loader/package/debian/tests/test_tasks.py index f55979f..5767da4 100644 --- a/swh/loader/package/debian/tests/test_tasks.py +++ b/swh/loader/package/debian/tests/test_tasks.py @@ -1,65 +1,41 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid import pytest from swh.scheduler.model import ListedOrigin, Lister -from swh.scheduler.utils import create_origin_task_dict - -@pytest.fixture(autouse=True) -def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): - pass +NAMESPACE = "swh.loader.package.debian" @pytest.fixture def debian_lister(): - return Lister(name="debian-lister", instance_name="example", id=uuid.uuid4()) + return Lister(name="debian", instance_name="example", id=uuid.uuid4()) @pytest.fixture def debian_listed_origin(debian_lister): return ListedOrigin( lister_id=debian_lister.id, url="https://debian.example.org/package", visit_type="debian", extra_loader_arguments={"packages": {}}, ) -def test_tasks_debian_loader(mocker, swh_scheduler_celery_app): - mock_load = mocker.patch("swh.loader.package.debian.loader.DebianLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.debian.tasks.LoadDebian", - kwargs=dict(url="some-url", packages={}), - ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} - - -def test_tasks_debian_loader_for_listed_origin( - mocker, swh_scheduler_celery_app, debian_lister, debian_listed_origin +def test_debian_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, + debian_lister, + debian_listed_origin, ): - mock_load = mocker.patch("swh.loader.package.debian.loader.DebianLoader.load") - mock_load.return_value = {"status": "eventful"} - - task_dict = create_origin_task_dict(debian_listed_origin, debian_lister) - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.debian.tasks.LoadDebian", - kwargs=task_dict["arguments"]["kwargs"], + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.DebianLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadDebian", + lister=debian_lister, + listed_origin=debian_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/deposit/tests/test_tasks.py b/swh/loader/package/deposit/tests/test_tasks.py index 1d82756..d56c169 100644 --- a/swh/loader/package/deposit/tests/test_tasks.py +++ b/swh/loader/package/deposit/tests/test_tasks.py @@ -1,80 +1,41 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid import pytest from swh.scheduler.model import ListedOrigin, Lister -from swh.scheduler.utils import create_origin_task_dict - -@pytest.fixture(autouse=True) -def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): - pass +NAMESPACE = "swh.loader.package.deposit" @pytest.fixture def deposit_lister(): - return Lister(name="deposit-lister", instance_name="example", id=uuid.uuid4()) + return Lister(name="deposit", instance_name="example", id=uuid.uuid4()) @pytest.fixture def deposit_listed_origin(deposit_lister): return ListedOrigin( lister_id=deposit_lister.id, url="https://example.org/project", visit_type="deposit", extra_loader_arguments={"deposit_id": "some-d-id"}, ) -def test_tasks_deposit_loader( - mocker, - swh_scheduler_celery_app, -): - mock_loader = mocker.patch( - "swh.loader.package.deposit.loader.DepositLoader.from_configfile" - ) - mock_loader.return_value = mock_loader - mock_loader.load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.deposit.tasks.LoadDeposit", - kwargs=dict( - url="some-url", - deposit_id="some-d-id", - ), - ) - assert res - res.wait() - assert res.successful() - assert mock_loader.called - assert res.result == {"status": "eventful"} - - -def test_tasks_deposit_loader_for_listed_origin( - mocker, - swh_scheduler_celery_app, +def test_deposit_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, deposit_lister, deposit_listed_origin, ): - mock_loader = mocker.patch( - "swh.loader.package.deposit.loader.DepositLoader.from_configfile" - ) - mock_loader.return_value = mock_loader - mock_loader.load.return_value = {"status": "eventful"} - - task_dict = create_origin_task_dict(deposit_listed_origin, deposit_lister) - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.deposit.tasks.LoadDeposit", - kwargs=task_dict["arguments"]["kwargs"], + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.DepositLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadDeposit", + lister=deposit_lister, + listed_origin=deposit_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_loader.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/golang/tests/test_tasks.py b/swh/loader/package/golang/tests/test_tasks.py index 18819b9..02b1295 100644 --- a/swh/loader/package/golang/tests/test_tasks.py +++ b/swh/loader/package/golang/tests/test_tasks.py @@ -1,21 +1,40 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid -def test_tasks_golang_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +import pytest + +from swh.scheduler.model import ListedOrigin, Lister + +NAMESPACE = "swh.loader.package.golang" + + +@pytest.fixture +def golang_lister(): + return Lister(name="golang", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def golang_listed_origin(golang_lister): + return ListedOrigin( + lister_id=golang_lister.id, + url="https://pkg.go.dev/golang.org/whatever/package", + visit_type="golang", + ) + + +def test_golang_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, + golang_lister, + golang_listed_origin, ): - mock_load = mocker.patch("swh.loader.package.golang.loader.GolangLoader.load") - mock_load.return_value = {"status": "eventful"} - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.golang.tasks.LoadGolang", - kwargs={"url": "https://pkg.go.dev/golang.org/whatever/package"}, + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.GolangLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadGolang", + lister=golang_lister, + listed_origin=golang_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/maven/tests/test_tasks.py b/swh/loader/package/maven/tests/test_tasks.py index 71773a5..9b0524b 100644 --- a/swh/loader/package/maven/tests/test_tasks.py +++ b/swh/loader/package/maven/tests/test_tasks.py @@ -1,86 +1,56 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid import pytest from swh.scheduler.model import ListedOrigin, Lister -from swh.scheduler.utils import create_origin_task_dict + +NAMESPACE = "swh.loader.package.maven" MVN_ARTIFACTS = [ { "time": 1626109619335, "url": "https://repo1.maven.org/maven2/al/aldi/sprova4j/0.1.0/" + "sprova4j-0.1.0.jar", "gid": "al.aldi", "aid": "sprova4j", "filename": "sprova4j-0.1.0.jar", "version": "0.1.0", "base_url": "https://repo1.maven.org/maven2/", }, ] -@pytest.fixture(autouse=True) -def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): - pass - - @pytest.fixture def maven_lister(): - return Lister(name="maven-lister", instance_name="example", id=uuid.uuid4()) + return Lister(name="maven", instance_name="example", id=uuid.uuid4()) @pytest.fixture def maven_listed_origin(maven_lister): return ListedOrigin( lister_id=maven_lister.id, url=MVN_ARTIFACTS[0]["url"], visit_type="maven", extra_loader_arguments={ "artifacts": MVN_ARTIFACTS, }, ) -def test_tasks_maven_loader( - mocker, - swh_scheduler_celery_app, +def test_maven_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, + maven_lister, + maven_listed_origin, ): - mock_load = mocker.patch("swh.loader.package.maven.loader.MavenLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.maven.tasks.LoadMaven", - kwargs=dict( - url=MVN_ARTIFACTS[0]["url"], - artifacts=MVN_ARTIFACTS, - ), - ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} - - -def test_tasks_maven_loader_for_listed_origin( - mocker, swh_scheduler_celery_app, maven_lister, maven_listed_origin -): - mock_load = mocker.patch("swh.loader.package.maven.loader.MavenLoader.load") - mock_load.return_value = {"status": "eventful"} - - task_dict = create_origin_task_dict(maven_listed_origin, maven_lister) - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.maven.tasks.LoadMaven", - kwargs=task_dict["arguments"]["kwargs"], + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.MavenLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadMaven", + lister=maven_lister, + listed_origin=maven_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/nixguix/tests/test_tasks.py b/swh/loader/package/nixguix/tests/test_tasks.py index 5249568..48cd4f9 100644 --- a/swh/loader/package/nixguix/tests/test_tasks.py +++ b/swh/loader/package/nixguix/tests/test_tasks.py @@ -1,72 +1,40 @@ # Copyright (C) 2020-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid import pytest from swh.scheduler.model import ListedOrigin, Lister -from swh.scheduler.utils import create_origin_task_dict - -@pytest.fixture(autouse=True) -def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): - pass +NAMESPACE = "swh.loader.package.nixguix" @pytest.fixture def nixguix_lister(): - return Lister(name="nixguix-lister", instance_name="example", id=uuid.uuid4()) + return Lister(name="nixguix", instance_name="example", id=uuid.uuid4()) @pytest.fixture def nixguix_listed_origin(nixguix_lister): return ListedOrigin( lister_id=nixguix_lister.id, url="https://nixguix.example.org/", visit_type="nixguix", ) -def test_tasks_nixguix_loader( - mocker, - swh_scheduler_celery_app, +def test_nixguix_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, + nixguix_lister, + nixguix_listed_origin, ): - mock_loader = mocker.patch( - "swh.loader.package.nixguix.loader.NixGuixLoader.from_configfile" - ) - mock_loader.return_value = mock_loader - mock_loader.load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.nixguix.tasks.LoadNixguix", kwargs=dict(url="some-url") - ) - assert res - res.wait() - assert res.successful() - assert mock_loader.called - assert res.result == {"status": "eventful"} - - -def test_tasks_nixguix_loader_for_listed_origin( - mocker, swh_scheduler_celery_app, nixguix_lister, nixguix_listed_origin -): - mock_loader = mocker.patch( - "swh.loader.package.nixguix.loader.NixGuixLoader.from_configfile" - ) - mock_loader.return_value = mock_loader - mock_loader.load.return_value = {"status": "eventful"} - - task_dict = create_origin_task_dict(nixguix_listed_origin, nixguix_lister) - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.nixguix.tasks.LoadNixguix", - kwargs=task_dict["arguments"]["kwargs"], + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.NixGuixLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadNixguix", + lister=nixguix_lister, + listed_origin=nixguix_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_loader.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/npm/tests/test_tasks.py b/swh/loader/package/npm/tests/test_tasks.py index 83b4d5d..650e1ae 100644 --- a/swh/loader/package/npm/tests/test_tasks.py +++ b/swh/loader/package/npm/tests/test_tasks.py @@ -1,67 +1,40 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid import pytest from swh.scheduler.model import ListedOrigin, Lister -from swh.scheduler.utils import create_origin_task_dict - -@pytest.fixture(autouse=True) -def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): - pass +NAMESPACE = "swh.loader.package.npm" @pytest.fixture def npm_lister(): - return Lister(name="npm-lister", instance_name="npm", id=uuid.uuid4()) + return Lister(name="npm", instance_name="npm", id=uuid.uuid4()) @pytest.fixture def npm_listed_origin(npm_lister): return ListedOrigin( lister_id=npm_lister.id, url="https://www.npmjs.com/package/some-package", visit_type="npm", ) -def test_tasks_npm_loader( - mocker, - swh_scheduler_celery_app, +def test_npm_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, + npm_lister, + npm_listed_origin, ): - mock_load = mocker.patch("swh.loader.package.npm.loader.NpmLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.npm.tasks.LoadNpm", - kwargs=dict(url="https://www.npmjs.com/package/some-package"), - ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} - - -def test_tasks_npm_loader_for_listed_origin( - mocker, swh_scheduler_celery_app, npm_lister, npm_listed_origin -): - mock_load = mocker.patch("swh.loader.package.npm.loader.NpmLoader.load") - mock_load.return_value = {"status": "eventful"} - - task_dict = create_origin_task_dict(npm_listed_origin, npm_lister) - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.npm.tasks.LoadNpm", - kwargs=task_dict["arguments"]["kwargs"], + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.NpmLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadNpm", + lister=npm_lister, + listed_origin=npm_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/opam/tests/test_tasks.py b/swh/loader/package/opam/tests/test_tasks.py index eec635e..96939c1 100644 --- a/swh/loader/package/opam/tests/test_tasks.py +++ b/swh/loader/package/opam/tests/test_tasks.py @@ -1,78 +1,51 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid import pytest from swh.scheduler.model import ListedOrigin, Lister -from swh.scheduler.utils import create_origin_task_dict + +NAMESPACE = "swh.loader.package.opam" OPAM_LOADER_ARGS = { "url": "opam+https://opam.ocaml.org/packages/agrid", "opam_root": "/tmp/test_tasks_opam_loader", "opam_instance": "test_tasks_opam_loader", "opam_url": "https://opam.ocaml.org", "opam_package": "agrid", } -@pytest.fixture(autouse=True) -def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): - pass - - @pytest.fixture def opam_lister(): - return Lister(name="opam-lister", instance_name="example", id=uuid.uuid4()) + return Lister(name="opam", instance_name="example", id=uuid.uuid4()) @pytest.fixture def opam_listed_origin(opam_lister): return ListedOrigin( lister_id=opam_lister.id, url=OPAM_LOADER_ARGS["url"], visit_type="opam", extra_loader_arguments={ k: v for k, v in OPAM_LOADER_ARGS.items() if k != "url" }, ) -def test_tasks_opam_loader( - mocker, - swh_scheduler_celery_app, +def test_opam_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, + opam_lister, + opam_listed_origin, ): - mock_load = mocker.patch("swh.loader.package.opam.loader.OpamLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.opam.tasks.LoadOpam", - kwargs=OPAM_LOADER_ARGS, - ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} - - -def test_tasks_opam_loader_for_listed_origin( - mocker, swh_scheduler_celery_app, opam_lister, opam_listed_origin -): - mock_load = mocker.patch("swh.loader.package.opam.loader.OpamLoader.load") - mock_load.return_value = {"status": "eventful"} - - task_dict = create_origin_task_dict(opam_listed_origin, opam_lister) - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.opam.tasks.LoadOpam", - kwargs=task_dict["arguments"]["kwargs"], + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.OpamLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadOpam", + lister=opam_lister, + listed_origin=opam_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/pubdev/tests/test_tasks.py b/swh/loader/package/pubdev/tests/test_tasks.py index c5b2ce7..c70777e 100644 --- a/swh/loader/package/pubdev/tests/test_tasks.py +++ b/swh/loader/package/pubdev/tests/test_tasks.py @@ -1,23 +1,40 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid -def test_tasks_pubdev_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +import pytest + +from swh.scheduler.model import ListedOrigin, Lister + +NAMESPACE = "swh.loader.package.pubdev" + + +@pytest.fixture +def pubdev_lister(): + return Lister(name="pubdev", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def pubdev_listed_origin(pubdev_lister): + return ListedOrigin( + lister_id=pubdev_lister.id, + url="https://pub.dev/packages/some-package", + visit_type="pubdev", + ) + + +def test_pubdev_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, + pubdev_lister, + pubdev_listed_origin, ): - mock_load = mocker.patch("swh.loader.package.pubdev.loader.PubDevLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.pubdev.tasks.LoadPubDev", - kwargs=dict( - url="https://pub.dev/packages/some-package", - ), + + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.PubDevLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadPubDev", + lister=pubdev_lister, + listed_origin=pubdev_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/puppet/tests/test_tasks.py b/swh/loader/package/puppet/tests/test_tasks.py index c8ba44f..e3a0d5a 100644 --- a/swh/loader/package/puppet/tests/test_tasks.py +++ b/swh/loader/package/puppet/tests/test_tasks.py @@ -1,31 +1,50 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid -def test_tasks_puppet_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config -): - mock_load = mocker.patch("swh.loader.package.puppet.loader.PuppetLoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.puppet.tasks.LoadPuppet", - kwargs=dict( - url="some-url/api/packages/some-package", - artifacts={ +import pytest + +from swh.scheduler.model import ListedOrigin, Lister + +NAMESPACE = "swh.loader.package.puppet" + + +@pytest.fixture +def puppet_lister(): + return Lister(name="puppet", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def puppet_listed_origin(puppet_lister): + return ListedOrigin( + lister_id=puppet_lister.id, + url="some-url/api/packages/some-package", + visit_type="aur", + extra_loader_arguments={ + "artifacts": { "1.0.0": { "url": "https://domain/some-package-1.0.0.tar.gz", "version": "1.0.0", "filename": "some-module-1.0.0.tar.gz", "last_update": "2011-11-20T13:40:30-08:00", }, - }, - ), + } + }, + ) + + +def test_puppet_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, + puppet_lister, + puppet_listed_origin, +): + + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.PuppetLoader", + task_function_name=f"{NAMESPACE}.tasks.LoadPuppet", + lister=puppet_lister, + listed_origin=puppet_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/pypi/tests/test_tasks.py b/swh/loader/package/pypi/tests/test_tasks.py index c294762..26f7abf 100644 --- a/swh/loader/package/pypi/tests/test_tasks.py +++ b/swh/loader/package/pypi/tests/test_tasks.py @@ -1,69 +1,40 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid import pytest from swh.scheduler.model import ListedOrigin, Lister -from swh.scheduler.utils import create_origin_task_dict - -@pytest.fixture(autouse=True) -def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): - pass +NAMESPACE = "swh.loader.package.pypi" @pytest.fixture def pypi_lister(): - return Lister(name="pypi-lister", instance_name="example", id=uuid.uuid4()) + return Lister(name="pypi", instance_name="example", id=uuid.uuid4()) @pytest.fixture def pypi_listed_origin(pypi_lister): return ListedOrigin( lister_id=pypi_lister.id, url="https://pypi.example.org/package", visit_type="pypi", ) -def test_tasks_pypi_loader( - mocker, - swh_scheduler_celery_app, -): - mock_load = mocker.patch("swh.loader.package.pypi.loader.PyPILoader.load") - mock_load.return_value = {"status": "eventful"} - - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.pypi.tasks.LoadPyPI", kwargs=dict(url="some-url") - ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} - - -def test_tasks_pypi_loader_for_listed_origin( - mocker, - swh_scheduler_celery_app, +def test_pypi_loader_task_for_listed_origin( + loading_task_creation_for_listed_origin_test, pypi_lister, pypi_listed_origin, ): - mock_load = mocker.patch("swh.loader.package.pypi.loader.PyPILoader.load") - mock_load.return_value = {"status": "eventful"} - - task_dict = create_origin_task_dict(pypi_listed_origin, pypi_lister) - res = swh_scheduler_celery_app.send_task( - "swh.loader.package.pypi.tasks.LoadPyPI", - kwargs=task_dict["arguments"]["kwargs"], + loading_task_creation_for_listed_origin_test( + loader_class_name=f"{NAMESPACE}.loader.PyPILoader", + task_function_name=f"{NAMESPACE}.tasks.LoadPyPI", + lister=pypi_lister, + listed_origin=pypi_listed_origin, ) - assert res - res.wait() - assert res.successful() - assert mock_load.called - assert res.result == {"status": "eventful"} diff --git a/swh/loader/pytest_plugin.py b/swh/loader/pytest_plugin.py index e501f02..57178b1 100644 --- a/swh/loader/pytest_plugin.py +++ b/swh/loader/pytest_plugin.py @@ -1,54 +1,88 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os from typing import Any, Dict import pytest import yaml +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + @pytest.fixture def swh_storage_backend_config(swh_storage_postgresql) -> Dict[str, Any]: return { "cls": "retry", "storage": { "cls": "filter", "storage": { "cls": "buffer", "storage": { "cls": "postgresql", "db": swh_storage_postgresql.dsn, "objstorage": {"cls": "memory"}, }, }, }, } @pytest.fixture def swh_loader_config(swh_storage_backend_config) -> Dict[str, Any]: return { "storage": swh_storage_backend_config, } @pytest.fixture def swh_config(swh_loader_config, monkeypatch, tmp_path) -> str: conffile = os.path.join(str(tmp_path), "loader.yml") with open(conffile, "w") as f: f.write(yaml.dump(swh_loader_config)) monkeypatch.setenv("SWH_CONFIG_FILENAME", conffile) return conffile @pytest.fixture(autouse=True, scope="session") def swh_proxy(): """Automatically inject this fixture in all tests to ensure no outside connection takes place. """ os.environ["http_proxy"] = "http://localhost:999" os.environ["https_proxy"] = "http://localhost:999" + + +@pytest.fixture +def loading_task_creation_for_listed_origin_test( + mocker, + swh_scheduler_celery_app, + swh_scheduler_celery_worker, + swh_config, +): + def test_implementation( + loader_class_name: str, + task_function_name: str, + lister: Lister, + listed_origin: ListedOrigin, + ): + mock_load = mocker.patch(f"{loader_class_name}.load") + mock_load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(listed_origin, lister) + + res = swh_scheduler_celery_app.send_task( + task_function_name, + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"} + + return test_implementation diff --git a/swh/loader/tests/conftest.py b/swh/loader/tests/conftest.py index 0494f2b..33b9eea 100644 --- a/swh/loader/tests/conftest.py +++ b/swh/loader/tests/conftest.py @@ -1,24 +1,24 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Any, Dict import pytest @pytest.fixture def swh_loader_config() -> Dict[str, Any]: return { "storage": { "cls": "memory", }, "deposit": { "url": "https://deposit.softwareheritage.org/1/private", "auth": { "username": "user", "password": "pass", }, }, }