diff --git a/swh/loader/package/archive/tasks.py b/swh/loader/package/archive/tasks.py --- a/swh/loader/package/archive/tasks.py +++ b/swh/loader/package/archive/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,9 +9,7 @@ @shared_task(name=__name__ + ".LoadArchive") -def load_archive_files(*, url=None, artifacts=None, snapshot_append=False): +def load_archive_files(**kwargs): """Load archive's artifacts (e.g gnu, etc...)""" - loader = ArchiveLoader.from_configfile( - url=url, artifacts=artifacts, snapshot_append=snapshot_append - ) + loader = ArchiveLoader.from_configfile(**kwargs) return loader.load() diff --git a/swh/loader/package/archive/tests/test_tasks.py b/swh/loader/package/archive/tests/test_tasks.py --- a/swh/loader/package/archive/tests/test_tasks.py +++ b/swh/loader/package/archive/tests/test_tasks.py @@ -1,11 +1,42 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def archive_lister(): + return Lister(name="archive-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def archive_listed_origin(archive_lister): + return ListedOrigin( + lister_id=archive_lister.id, + url="https://example.org/archives", + visit_type="tar", + extra_loader_arguments={ + "artifacts": [], + "snapshot_append": True, + }, + ) + def test_tasks_archive_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_load = mocker.patch("swh.loader.package.archive.loader.ArchiveLoader.load") mock_load.return_value = {"status": "eventful"} @@ -22,7 +53,8 @@ def test_tasks_archive_loader_snapshot_append( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_load = mocker.patch("swh.loader.package.archive.loader.ArchiveLoader.load") mock_load.return_value = {"status": "eventful"} @@ -36,3 +68,25 @@ assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} + + +def test_tasks_archive_loader_for_listed_origin( + mocker, + swh_scheduler_celery_app, + archive_lister, + archive_listed_origin, +): + mock_load = mocker.patch("swh.loader.package.archive.loader.ArchiveLoader.load") + mock_load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(archive_listed_origin, archive_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.archive.tasks.LoadArchive", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/cran/tasks.py b/swh/loader/package/cran/tasks.py --- a/swh/loader/package/cran/tasks.py +++ b/swh/loader/package/cran/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,6 +9,6 @@ @shared_task(name=__name__ + ".LoadCRAN") -def load_cran(url=None, artifacts=[]): +def load_cran(**kwargs): """Load CRAN's artifacts""" - return CRANLoader.from_configfile(url=url, artifacts=artifacts).load() + return CRANLoader.from_configfile(**kwargs).load() diff --git a/swh/loader/package/cran/tests/test_tasks.py b/swh/loader/package/cran/tests/test_tasks.py --- a/swh/loader/package/cran/tests/test_tasks.py +++ b/swh/loader/package/cran/tests/test_tasks.py @@ -1,11 +1,41 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def cran_lister(): + return Lister(name="cran-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def cran_listed_origin(cran_lister): + return ListedOrigin( + lister_id=cran_lister.id, + url="https://cran.example.org/project", + visit_type="cran", + extra_loader_arguments={ + "artifacts": [{"version": "1.2.3", "url": "artifact-url"}], + }, + ) + def test_tasks_cran_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_load = mocker.patch("swh.loader.package.cran.loader.CRANLoader.load") mock_load.return_value = {"status": "eventful"} @@ -22,3 +52,25 @@ assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} + + +def test_tasks_cran_loader_for_listed_origin( + mocker, + swh_scheduler_celery_app, + cran_lister, + cran_listed_origin, +): + mock_load = mocker.patch("swh.loader.package.cran.loader.CRANLoader.load") + mock_load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(cran_listed_origin, cran_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.cran.tasks.LoadCRAN", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/debian/tasks.py b/swh/loader/package/debian/tasks.py --- a/swh/loader/package/debian/tasks.py +++ b/swh/loader/package/debian/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,7 +9,7 @@ @shared_task(name=__name__ + ".LoadDebian") -def load_deb(*, url, packages): +def load_deb(**kwargs): """Load Debian package""" - loader = DebianLoader.from_configfile(url=url, packages=packages) + loader = DebianLoader.from_configfile(**kwargs) return loader.load() diff --git a/swh/loader/package/debian/tests/test_tasks.py b/swh/loader/package/debian/tests/test_tasks.py --- a/swh/loader/package/debian/tests/test_tasks.py +++ b/swh/loader/package/debian/tests/test_tasks.py @@ -1,12 +1,37 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid -def test_tasks_debian_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config -): +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def debian_lister(): + return Lister(name="debian-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def debian_listed_origin(debian_lister): + return ListedOrigin( + lister_id=debian_lister.id, + url="https://debian.example.org/package", + visit_type="debian", + extra_loader_arguments={"packages": {}}, + ) + + +def test_tasks_debian_loader(mocker, swh_scheduler_celery_app): mock_load = mocker.patch("swh.loader.package.debian.loader.DebianLoader.load") mock_load.return_value = {"status": "eventful"} @@ -19,3 +44,22 @@ assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} + + +def test_tasks_debian_loader_for_listed_origin( + mocker, swh_scheduler_celery_app, debian_lister, debian_listed_origin +): + mock_load = mocker.patch("swh.loader.package.debian.loader.DebianLoader.load") + mock_load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(debian_listed_origin, debian_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.debian.tasks.LoadDebian", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/deposit/tasks.py b/swh/loader/package/deposit/tasks.py --- a/swh/loader/package/deposit/tasks.py +++ b/swh/loader/package/deposit/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,6 +9,6 @@ @shared_task(name=__name__ + ".LoadDeposit") -def load_deposit(*, url, deposit_id): +def load_deposit(**kwargs): """Load Deposit artifacts""" - return DepositLoader.from_configfile(url=url, deposit_id=deposit_id).load() + return DepositLoader.from_configfile(**kwargs).load() diff --git a/swh/loader/package/deposit/tests/test_tasks.py b/swh/loader/package/deposit/tests/test_tasks.py --- a/swh/loader/package/deposit/tests/test_tasks.py +++ b/swh/loader/package/deposit/tests/test_tasks.py @@ -1,11 +1,39 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def deposit_lister(): + return Lister(name="deposit-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def deposit_listed_origin(deposit_lister): + return ListedOrigin( + lister_id=deposit_lister.id, + url="https://example.org/project", + visit_type="deposit", + extra_loader_arguments={"deposit_id": "some-d-id"}, + ) + def test_tasks_deposit_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_loader = mocker.patch( "swh.loader.package.deposit.loader.DepositLoader.from_configfile" @@ -25,3 +53,28 @@ assert res.successful() assert mock_loader.called assert res.result == {"status": "eventful"} + + +def test_tasks_deposit_loader_for_listed_origin( + mocker, + swh_scheduler_celery_app, + deposit_lister, + deposit_listed_origin, +): + mock_loader = mocker.patch( + "swh.loader.package.deposit.loader.DepositLoader.from_configfile" + ) + mock_loader.return_value = mock_loader + mock_loader.load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(deposit_listed_origin, deposit_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.deposit.tasks.LoadDeposit", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_loader.called + assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/maven/tasks.py b/swh/loader/package/maven/tasks.py --- a/swh/loader/package/maven/tasks.py +++ b/swh/loader/package/maven/tasks.py @@ -9,7 +9,7 @@ @shared_task(name=__name__ + ".LoadMaven") -def load_maven(*, url=None, artifacts=None): +def load_maven(**kwargs): """Load maven jar artifacts.""" - loader = MavenLoader.from_configfile(url=url, artifacts=artifacts) + loader = MavenLoader.from_configfile(**kwargs) return loader.load() diff --git a/swh/loader/package/maven/tests/test_tasks.py b/swh/loader/package/maven/tests/test_tasks.py --- a/swh/loader/package/maven/tests/test_tasks.py +++ b/swh/loader/package/maven/tests/test_tasks.py @@ -1,8 +1,15 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + MVN_ARTIFACTS = [ { "time": 1626109619335, @@ -17,8 +24,31 @@ ] +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def maven_lister(): + return Lister(name="maven-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def maven_listed_origin(maven_lister): + return ListedOrigin( + lister_id=maven_lister.id, + url=MVN_ARTIFACTS[0]["url"], + visit_type="maven", + extra_loader_arguments={ + "artifacts": MVN_ARTIFACTS, + }, + ) + + def test_tasks_maven_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_load = mocker.patch("swh.loader.package.maven.loader.MavenLoader.load") mock_load.return_value = {"status": "eventful"} @@ -37,15 +67,17 @@ assert res.result == {"status": "eventful"} -def test_tasks_maven_loader_snapshot_append( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +def test_tasks_maven_loader_for_listed_origin( + mocker, swh_scheduler_celery_app, maven_lister, maven_listed_origin ): mock_load = mocker.patch("swh.loader.package.maven.loader.MavenLoader.load") mock_load.return_value = {"status": "eventful"} + task_dict = create_origin_task_dict(maven_listed_origin, maven_lister) + res = swh_scheduler_celery_app.send_task( "swh.loader.package.maven.tasks.LoadMaven", - kwargs=dict(url=MVN_ARTIFACTS[0]["url"], artifacts=[]), + kwargs=task_dict["arguments"]["kwargs"], ) assert res res.wait() diff --git a/swh/loader/package/nixguix/tasks.py b/swh/loader/package/nixguix/tasks.py --- a/swh/loader/package/nixguix/tasks.py +++ b/swh/loader/package/nixguix/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2021 The Software Heritage developers +# Copyright (C) 2020-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,6 +9,6 @@ @shared_task(name=__name__ + ".LoadNixguix") -def load_nixguix(*, url=None): +def load_nixguix(**kwargs): """Load functional (e.g. guix/nix) package""" - return NixGuixLoader.from_configfile(url=url).load() + return NixGuixLoader.from_configfile(**kwargs).load() diff --git a/swh/loader/package/nixguix/tests/test_tasks.py b/swh/loader/package/nixguix/tests/test_tasks.py --- a/swh/loader/package/nixguix/tests/test_tasks.py +++ b/swh/loader/package/nixguix/tests/test_tasks.py @@ -1,11 +1,38 @@ -# Copyright (C) 2020-2021 The Software Heritage developers +# Copyright (C) 2020-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def nixguix_lister(): + return Lister(name="nixguix-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def nixguix_listed_origin(nixguix_lister): + return ListedOrigin( + lister_id=nixguix_lister.id, + url="https://nixguix.example.org/", + visit_type="nixguix", + ) + def test_tasks_nixguix_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_loader = mocker.patch( "swh.loader.package.nixguix.loader.NixGuixLoader.from_configfile" @@ -21,3 +48,25 @@ assert res.successful() assert mock_loader.called assert res.result == {"status": "eventful"} + + +def test_tasks_nixguix_loader_for_listed_origin( + mocker, swh_scheduler_celery_app, nixguix_lister, nixguix_listed_origin +): + mock_loader = mocker.patch( + "swh.loader.package.nixguix.loader.NixGuixLoader.from_configfile" + ) + mock_loader.return_value = mock_loader + mock_loader.load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(nixguix_listed_origin, nixguix_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.nixguix.tasks.LoadNixguix", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_loader.called + assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/npm/tasks.py b/swh/loader/package/npm/tasks.py --- a/swh/loader/package/npm/tasks.py +++ b/swh/loader/package/npm/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,6 +9,6 @@ @shared_task(name=__name__ + ".LoadNpm") -def load_npm(*, url: str): +def load_npm(**kwargs): """Load Npm package""" - return NpmLoader.from_configfile(url=url).load() + return NpmLoader.from_configfile(**kwargs).load() diff --git a/swh/loader/package/npm/tests/test_tasks.py b/swh/loader/package/npm/tests/test_tasks.py --- a/swh/loader/package/npm/tests/test_tasks.py +++ b/swh/loader/package/npm/tests/test_tasks.py @@ -1,11 +1,38 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def npm_lister(): + return Lister(name="npm-lister", instance_name="npm", id=uuid.uuid4()) + + +@pytest.fixture +def npm_listed_origin(npm_lister): + return ListedOrigin( + lister_id=npm_lister.id, + url="https://www.npmjs.com/package/some-package", + visit_type="npm", + ) + def test_tasks_npm_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_load = mocker.patch("swh.loader.package.npm.loader.NpmLoader.load") mock_load.return_value = {"status": "eventful"} @@ -19,3 +46,22 @@ assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} + + +def test_tasks_npm_loader_for_listed_origin( + mocker, swh_scheduler_celery_app, npm_lister, npm_listed_origin +): + mock_load = mocker.patch("swh.loader.package.npm.loader.NpmLoader.load") + mock_load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(npm_listed_origin, npm_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.npm.tasks.LoadNpm", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/opam/tasks.py b/swh/loader/package/opam/tasks.py --- a/swh/loader/package/opam/tasks.py +++ b/swh/loader/package/opam/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021 The Software Heritage developers +# Copyright (C) 2021-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,12 +9,6 @@ @shared_task(name=__name__ + ".LoadOpam") -def load_opam(url, opam_root, opam_instance, opam_url, opam_package): +def load_opam(**kwargs): """Load Opam's artifacts""" - return OpamLoader.from_configfile( - url=url, - opam_root=opam_root, - opam_instance=opam_instance, - opam_url=opam_url, - opam_package=opam_package, - ).load() + return OpamLoader.from_configfile(**kwargs).load() diff --git a/swh/loader/package/opam/tests/test_tasks.py b/swh/loader/package/opam/tests/test_tasks.py --- a/swh/loader/package/opam/tests/test_tasks.py +++ b/swh/loader/package/opam/tests/test_tasks.py @@ -1,24 +1,75 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + +OPAM_LOADER_ARGS = { + "url": "opam+https://opam.ocaml.org/packages/agrid", + "opam_root": "/tmp/test_tasks_opam_loader", + "opam_instance": "test_tasks_opam_loader", + "opam_url": "https://opam.ocaml.org", + "opam_package": "agrid", +} + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def opam_lister(): + return Lister(name="opam-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def opam_listed_origin(opam_lister): + return ListedOrigin( + lister_id=opam_lister.id, + url=OPAM_LOADER_ARGS["url"], + visit_type="opam", + extra_loader_arguments={ + k: v for k, v in OPAM_LOADER_ARGS.items() if k != "url" + }, + ) + def test_tasks_opam_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, +): + mock_load = mocker.patch("swh.loader.package.opam.loader.OpamLoader.load") + mock_load.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.opam.tasks.LoadOpam", + kwargs=OPAM_LOADER_ARGS, + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"} + + +def test_tasks_opam_loader_for_listed_origin( + mocker, swh_scheduler_celery_app, opam_lister, opam_listed_origin ): mock_load = mocker.patch("swh.loader.package.opam.loader.OpamLoader.load") mock_load.return_value = {"status": "eventful"} + task_dict = create_origin_task_dict(opam_listed_origin, opam_lister) + res = swh_scheduler_celery_app.send_task( "swh.loader.package.opam.tasks.LoadOpam", - args=( - "opam+https://opam.ocaml.org/packages/agrid", # url - "/tmp/test_tasks_opam_loader", # opam_root - "test_tasks_opam_loader", # opam_instance - "https://opam.ocaml.org", # opam_url - "agrid", # opam_package - ), + kwargs=task_dict["arguments"]["kwargs"], ) assert res res.wait() diff --git a/swh/loader/package/pypi/tasks.py b/swh/loader/package/pypi/tasks.py --- a/swh/loader/package/pypi/tasks.py +++ b/swh/loader/package/pypi/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,6 +9,6 @@ @shared_task(name=__name__ + ".LoadPyPI") -def load_pypi(*, url=None): +def load_pypi(**kwargs): """Load PyPI package""" - return PyPILoader.from_configfile(url=url).load() + return PyPILoader.from_configfile(**kwargs).load() diff --git a/swh/loader/package/pypi/tests/test_tasks.py b/swh/loader/package/pypi/tests/test_tasks.py --- a/swh/loader/package/pypi/tests/test_tasks.py +++ b/swh/loader/package/pypi/tests/test_tasks.py @@ -1,11 +1,38 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def pypi_lister(): + return Lister(name="pypi-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def pypi_listed_origin(pypi_lister): + return ListedOrigin( + lister_id=pypi_lister.id, + url="https://pypi.example.org/package", + visit_type="pypi", + ) + def test_tasks_pypi_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_load = mocker.patch("swh.loader.package.pypi.loader.PyPILoader.load") mock_load.return_value = {"status": "eventful"} @@ -18,3 +45,25 @@ assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} + + +def test_tasks_pypi_loader_for_listed_origin( + mocker, + swh_scheduler_celery_app, + pypi_lister, + pypi_listed_origin, +): + mock_load = mocker.patch("swh.loader.package.pypi.loader.PyPILoader.load") + mock_load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(pypi_listed_origin, pypi_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.pypi.tasks.LoadPyPI", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"}