diff --git a/swh/loader/package/archive/tasks.py b/swh/loader/package/archive/tasks.py index 727cffd..31a7b58 100644 --- a/swh/loader/package/archive/tasks.py +++ b/swh/loader/package/archive/tasks.py @@ -1,17 +1,15 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import shared_task from swh.loader.package.archive.loader import ArchiveLoader @shared_task(name=__name__ + ".LoadArchive") -def load_archive_files(*, url=None, artifacts=None, snapshot_append=False): +def load_archive_files(**kwargs): """Load archive's artifacts (e.g gnu, etc...)""" - loader = ArchiveLoader.from_configfile( - url=url, artifacts=artifacts, snapshot_append=snapshot_append - ) + loader = ArchiveLoader.from_configfile(**kwargs) return loader.load() diff --git a/swh/loader/package/archive/tests/test_tasks.py b/swh/loader/package/archive/tests/test_tasks.py index 74e1967..0d2448f 100644 --- a/swh/loader/package/archive/tests/test_tasks.py +++ b/swh/loader/package/archive/tests/test_tasks.py @@ -1,38 +1,92 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def archive_lister(): + return Lister(name="archive-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def archive_listed_origin(archive_lister): + return ListedOrigin( + lister_id=archive_lister.id, + url="https://example.org/archives", + visit_type="tar", + extra_loader_arguments={ + "artifacts": [], + "snapshot_append": True, + }, + ) + def test_tasks_archive_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_load = mocker.patch("swh.loader.package.archive.loader.ArchiveLoader.load") mock_load.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.package.archive.tasks.LoadArchive", kwargs=dict(url="https://gnu.org/", artifacts=[]), ) assert res res.wait() assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} def test_tasks_archive_loader_snapshot_append( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_load = mocker.patch("swh.loader.package.archive.loader.ArchiveLoader.load") mock_load.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.package.archive.tasks.LoadArchive", kwargs=dict(url="https://gnu.org/", artifacts=[], snapshot_append=True), ) assert res res.wait() assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} + + +def test_tasks_archive_loader_for_listed_origin( + mocker, + swh_scheduler_celery_app, + archive_lister, + archive_listed_origin, +): + mock_load = mocker.patch("swh.loader.package.archive.loader.ArchiveLoader.load") + mock_load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(archive_listed_origin, archive_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.archive.tasks.LoadArchive", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/cran/tasks.py b/swh/loader/package/cran/tasks.py index ac5c53a..8fa82f8 100644 --- a/swh/loader/package/cran/tasks.py +++ b/swh/loader/package/cran/tasks.py @@ -1,14 +1,14 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import shared_task from swh.loader.package.cran.loader import CRANLoader @shared_task(name=__name__ + ".LoadCRAN") -def load_cran(url=None, artifacts=[]): +def load_cran(**kwargs): """Load CRAN's artifacts""" - return CRANLoader.from_configfile(url=url, artifacts=artifacts).load() + return CRANLoader.from_configfile(**kwargs).load() diff --git a/swh/loader/package/cran/tests/test_tasks.py b/swh/loader/package/cran/tests/test_tasks.py index ae8a604..e2b5de2 100644 --- a/swh/loader/package/cran/tests/test_tasks.py +++ b/swh/loader/package/cran/tests/test_tasks.py @@ -1,24 +1,76 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def cran_lister(): + return Lister(name="cran-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def cran_listed_origin(cran_lister): + return ListedOrigin( + lister_id=cran_lister.id, + url="https://cran.example.org/project", + visit_type="cran", + extra_loader_arguments={ + "artifacts": [{"version": "1.2.3", "url": "artifact-url"}], + }, + ) + def test_tasks_cran_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_load = mocker.patch("swh.loader.package.cran.loader.CRANLoader.load") mock_load.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.package.cran.tasks.LoadCRAN", kwargs=dict( url="some-url", artifacts=[{"version": "1.2.3", "url": "artifact-url"}], ), ) assert res res.wait() assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} + + +def test_tasks_cran_loader_for_listed_origin( + mocker, + swh_scheduler_celery_app, + cran_lister, + cran_listed_origin, +): + mock_load = mocker.patch("swh.loader.package.cran.loader.CRANLoader.load") + mock_load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(cran_listed_origin, cran_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.cran.tasks.LoadCRAN", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/debian/tasks.py b/swh/loader/package/debian/tasks.py index c821ea3..38d582b 100644 --- a/swh/loader/package/debian/tasks.py +++ b/swh/loader/package/debian/tasks.py @@ -1,15 +1,15 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import shared_task from swh.loader.package.debian.loader import DebianLoader @shared_task(name=__name__ + ".LoadDebian") -def load_deb(*, url, packages): +def load_deb(**kwargs): """Load Debian package""" - loader = DebianLoader.from_configfile(url=url, packages=packages) + loader = DebianLoader.from_configfile(**kwargs) return loader.load() diff --git a/swh/loader/package/debian/tests/test_tasks.py b/swh/loader/package/debian/tests/test_tasks.py index 048953d..f55979f 100644 --- a/swh/loader/package/debian/tests/test_tasks.py +++ b/swh/loader/package/debian/tests/test_tasks.py @@ -1,21 +1,65 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid -def test_tasks_debian_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config -): +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def debian_lister(): + return Lister(name="debian-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def debian_listed_origin(debian_lister): + return ListedOrigin( + lister_id=debian_lister.id, + url="https://debian.example.org/package", + visit_type="debian", + extra_loader_arguments={"packages": {}}, + ) + + +def test_tasks_debian_loader(mocker, swh_scheduler_celery_app): mock_load = mocker.patch("swh.loader.package.debian.loader.DebianLoader.load") mock_load.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.package.debian.tasks.LoadDebian", kwargs=dict(url="some-url", packages={}), ) assert res res.wait() assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} + + +def test_tasks_debian_loader_for_listed_origin( + mocker, swh_scheduler_celery_app, debian_lister, debian_listed_origin +): + mock_load = mocker.patch("swh.loader.package.debian.loader.DebianLoader.load") + mock_load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(debian_listed_origin, debian_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.debian.tasks.LoadDebian", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/deposit/tasks.py b/swh/loader/package/deposit/tasks.py index 187651b..6372ff2 100644 --- a/swh/loader/package/deposit/tasks.py +++ b/swh/loader/package/deposit/tasks.py @@ -1,14 +1,14 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import shared_task from swh.loader.package.deposit.loader import DepositLoader @shared_task(name=__name__ + ".LoadDeposit") -def load_deposit(*, url, deposit_id): +def load_deposit(**kwargs): """Load Deposit artifacts""" - return DepositLoader.from_configfile(url=url, deposit_id=deposit_id).load() + return DepositLoader.from_configfile(**kwargs).load() diff --git a/swh/loader/package/deposit/tests/test_tasks.py b/swh/loader/package/deposit/tests/test_tasks.py index cd63efd..1d82756 100644 --- a/swh/loader/package/deposit/tests/test_tasks.py +++ b/swh/loader/package/deposit/tests/test_tasks.py @@ -1,27 +1,80 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def deposit_lister(): + return Lister(name="deposit-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def deposit_listed_origin(deposit_lister): + return ListedOrigin( + lister_id=deposit_lister.id, + url="https://example.org/project", + visit_type="deposit", + extra_loader_arguments={"deposit_id": "some-d-id"}, + ) + def test_tasks_deposit_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_loader = mocker.patch( "swh.loader.package.deposit.loader.DepositLoader.from_configfile" ) mock_loader.return_value = mock_loader mock_loader.load.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.package.deposit.tasks.LoadDeposit", kwargs=dict( url="some-url", deposit_id="some-d-id", ), ) assert res res.wait() assert res.successful() assert mock_loader.called assert res.result == {"status": "eventful"} + + +def test_tasks_deposit_loader_for_listed_origin( + mocker, + swh_scheduler_celery_app, + deposit_lister, + deposit_listed_origin, +): + mock_loader = mocker.patch( + "swh.loader.package.deposit.loader.DepositLoader.from_configfile" + ) + mock_loader.return_value = mock_loader + mock_loader.load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(deposit_listed_origin, deposit_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.deposit.tasks.LoadDeposit", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_loader.called + assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/maven/tasks.py b/swh/loader/package/maven/tasks.py index 49d2b0b..27fe787 100644 --- a/swh/loader/package/maven/tasks.py +++ b/swh/loader/package/maven/tasks.py @@ -1,15 +1,15 @@ # Copyright (C) 2021-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import shared_task from swh.loader.package.maven.loader import MavenLoader @shared_task(name=__name__ + ".LoadMaven") -def load_maven(*, url=None, artifacts=None): +def load_maven(**kwargs): """Load maven jar artifacts.""" - loader = MavenLoader.from_configfile(url=url, artifacts=artifacts) + loader = MavenLoader.from_configfile(**kwargs) return loader.load() diff --git a/swh/loader/package/maven/tests/test_tasks.py b/swh/loader/package/maven/tests/test_tasks.py index 479dce0..71773a5 100644 --- a/swh/loader/package/maven/tests/test_tasks.py +++ b/swh/loader/package/maven/tests/test_tasks.py @@ -1,54 +1,86 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + MVN_ARTIFACTS = [ { "time": 1626109619335, "url": "https://repo1.maven.org/maven2/al/aldi/sprova4j/0.1.0/" + "sprova4j-0.1.0.jar", "gid": "al.aldi", "aid": "sprova4j", "filename": "sprova4j-0.1.0.jar", "version": "0.1.0", "base_url": "https://repo1.maven.org/maven2/", }, ] +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def maven_lister(): + return Lister(name="maven-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def maven_listed_origin(maven_lister): + return ListedOrigin( + lister_id=maven_lister.id, + url=MVN_ARTIFACTS[0]["url"], + visit_type="maven", + extra_loader_arguments={ + "artifacts": MVN_ARTIFACTS, + }, + ) + + def test_tasks_maven_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_load = mocker.patch("swh.loader.package.maven.loader.MavenLoader.load") mock_load.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.package.maven.tasks.LoadMaven", kwargs=dict( url=MVN_ARTIFACTS[0]["url"], artifacts=MVN_ARTIFACTS, ), ) assert res res.wait() assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} -def test_tasks_maven_loader_snapshot_append( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +def test_tasks_maven_loader_for_listed_origin( + mocker, swh_scheduler_celery_app, maven_lister, maven_listed_origin ): mock_load = mocker.patch("swh.loader.package.maven.loader.MavenLoader.load") mock_load.return_value = {"status": "eventful"} + task_dict = create_origin_task_dict(maven_listed_origin, maven_lister) + res = swh_scheduler_celery_app.send_task( "swh.loader.package.maven.tasks.LoadMaven", - kwargs=dict(url=MVN_ARTIFACTS[0]["url"], artifacts=[]), + kwargs=task_dict["arguments"]["kwargs"], ) assert res res.wait() assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/nixguix/tasks.py b/swh/loader/package/nixguix/tasks.py index 39ddf48..44abd63 100644 --- a/swh/loader/package/nixguix/tasks.py +++ b/swh/loader/package/nixguix/tasks.py @@ -1,14 +1,14 @@ -# Copyright (C) 2020-2021 The Software Heritage developers +# Copyright (C) 2020-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import shared_task from swh.loader.package.nixguix.loader import NixGuixLoader @shared_task(name=__name__ + ".LoadNixguix") -def load_nixguix(*, url=None): +def load_nixguix(**kwargs): """Load functional (e.g. guix/nix) package""" - return NixGuixLoader.from_configfile(url=url).load() + return NixGuixLoader.from_configfile(**kwargs).load() diff --git a/swh/loader/package/nixguix/tests/test_tasks.py b/swh/loader/package/nixguix/tests/test_tasks.py index edb06e2..5249568 100644 --- a/swh/loader/package/nixguix/tests/test_tasks.py +++ b/swh/loader/package/nixguix/tests/test_tasks.py @@ -1,23 +1,72 @@ -# Copyright (C) 2020-2021 The Software Heritage developers +# Copyright (C) 2020-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def nixguix_lister(): + return Lister(name="nixguix-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def nixguix_listed_origin(nixguix_lister): + return ListedOrigin( + lister_id=nixguix_lister.id, + url="https://nixguix.example.org/", + visit_type="nixguix", + ) + def test_tasks_nixguix_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_loader = mocker.patch( "swh.loader.package.nixguix.loader.NixGuixLoader.from_configfile" ) mock_loader.return_value = mock_loader mock_loader.load.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.package.nixguix.tasks.LoadNixguix", kwargs=dict(url="some-url") ) assert res res.wait() assert res.successful() assert mock_loader.called assert res.result == {"status": "eventful"} + + +def test_tasks_nixguix_loader_for_listed_origin( + mocker, swh_scheduler_celery_app, nixguix_lister, nixguix_listed_origin +): + mock_loader = mocker.patch( + "swh.loader.package.nixguix.loader.NixGuixLoader.from_configfile" + ) + mock_loader.return_value = mock_loader + mock_loader.load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(nixguix_listed_origin, nixguix_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.nixguix.tasks.LoadNixguix", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_loader.called + assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/npm/tasks.py b/swh/loader/package/npm/tasks.py index 43fc0bf..991a349 100644 --- a/swh/loader/package/npm/tasks.py +++ b/swh/loader/package/npm/tasks.py @@ -1,14 +1,14 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import shared_task from swh.loader.package.npm.loader import NpmLoader @shared_task(name=__name__ + ".LoadNpm") -def load_npm(*, url: str): +def load_npm(**kwargs): """Load Npm package""" - return NpmLoader.from_configfile(url=url).load() + return NpmLoader.from_configfile(**kwargs).load() diff --git a/swh/loader/package/npm/tests/test_tasks.py b/swh/loader/package/npm/tests/test_tasks.py index 4cdbb36..83b4d5d 100644 --- a/swh/loader/package/npm/tests/test_tasks.py +++ b/swh/loader/package/npm/tests/test_tasks.py @@ -1,21 +1,67 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def npm_lister(): + return Lister(name="npm-lister", instance_name="npm", id=uuid.uuid4()) + + +@pytest.fixture +def npm_listed_origin(npm_lister): + return ListedOrigin( + lister_id=npm_lister.id, + url="https://www.npmjs.com/package/some-package", + visit_type="npm", + ) + def test_tasks_npm_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_load = mocker.patch("swh.loader.package.npm.loader.NpmLoader.load") mock_load.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.package.npm.tasks.LoadNpm", kwargs=dict(url="https://www.npmjs.com/package/some-package"), ) assert res res.wait() assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} + + +def test_tasks_npm_loader_for_listed_origin( + mocker, swh_scheduler_celery_app, npm_lister, npm_listed_origin +): + mock_load = mocker.patch("swh.loader.package.npm.loader.NpmLoader.load") + mock_load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(npm_listed_origin, npm_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.npm.tasks.LoadNpm", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/opam/tasks.py b/swh/loader/package/opam/tasks.py index 2ca7f7a..28417a6 100644 --- a/swh/loader/package/opam/tasks.py +++ b/swh/loader/package/opam/tasks.py @@ -1,20 +1,14 @@ -# Copyright (C) 2021 The Software Heritage developers +# Copyright (C) 2021-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import shared_task from swh.loader.package.opam.loader import OpamLoader @shared_task(name=__name__ + ".LoadOpam") -def load_opam(url, opam_root, opam_instance, opam_url, opam_package): +def load_opam(**kwargs): """Load Opam's artifacts""" - return OpamLoader.from_configfile( - url=url, - opam_root=opam_root, - opam_instance=opam_instance, - opam_url=opam_url, - opam_package=opam_package, - ).load() + return OpamLoader.from_configfile(**kwargs).load() diff --git a/swh/loader/package/opam/tests/test_tasks.py b/swh/loader/package/opam/tests/test_tasks.py index 8fe996c..eec635e 100644 --- a/swh/loader/package/opam/tests/test_tasks.py +++ b/swh/loader/package/opam/tests/test_tasks.py @@ -1,27 +1,78 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + +OPAM_LOADER_ARGS = { + "url": "opam+https://opam.ocaml.org/packages/agrid", + "opam_root": "/tmp/test_tasks_opam_loader", + "opam_instance": "test_tasks_opam_loader", + "opam_url": "https://opam.ocaml.org", + "opam_package": "agrid", +} + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def opam_lister(): + return Lister(name="opam-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def opam_listed_origin(opam_lister): + return ListedOrigin( + lister_id=opam_lister.id, + url=OPAM_LOADER_ARGS["url"], + visit_type="opam", + extra_loader_arguments={ + k: v for k, v in OPAM_LOADER_ARGS.items() if k != "url" + }, + ) + def test_tasks_opam_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, +): + mock_load = mocker.patch("swh.loader.package.opam.loader.OpamLoader.load") + mock_load.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.opam.tasks.LoadOpam", + kwargs=OPAM_LOADER_ARGS, + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"} + + +def test_tasks_opam_loader_for_listed_origin( + mocker, swh_scheduler_celery_app, opam_lister, opam_listed_origin ): mock_load = mocker.patch("swh.loader.package.opam.loader.OpamLoader.load") mock_load.return_value = {"status": "eventful"} + task_dict = create_origin_task_dict(opam_listed_origin, opam_lister) + res = swh_scheduler_celery_app.send_task( "swh.loader.package.opam.tasks.LoadOpam", - args=( - "opam+https://opam.ocaml.org/packages/agrid", # url - "/tmp/test_tasks_opam_loader", # opam_root - "test_tasks_opam_loader", # opam_instance - "https://opam.ocaml.org", # opam_url - "agrid", # opam_package - ), + kwargs=task_dict["arguments"]["kwargs"], ) assert res res.wait() assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} diff --git a/swh/loader/package/pypi/tasks.py b/swh/loader/package/pypi/tasks.py index 45a60c3..1917e54 100644 --- a/swh/loader/package/pypi/tasks.py +++ b/swh/loader/package/pypi/tasks.py @@ -1,14 +1,14 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import shared_task from swh.loader.package.pypi.loader import PyPILoader @shared_task(name=__name__ + ".LoadPyPI") -def load_pypi(*, url=None): +def load_pypi(**kwargs): """Load PyPI package""" - return PyPILoader.from_configfile(url=url).load() + return PyPILoader.from_configfile(**kwargs).load() diff --git a/swh/loader/package/pypi/tests/test_tasks.py b/swh/loader/package/pypi/tests/test_tasks.py index 35387b0..c294762 100644 --- a/swh/loader/package/pypi/tests/test_tasks.py +++ b/swh/loader/package/pypi/tests/test_tasks.py @@ -1,20 +1,69 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def pypi_lister(): + return Lister(name="pypi-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def pypi_listed_origin(pypi_lister): + return ListedOrigin( + lister_id=pypi_lister.id, + url="https://pypi.example.org/package", + visit_type="pypi", + ) + def test_tasks_pypi_loader( - mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config + mocker, + swh_scheduler_celery_app, ): mock_load = mocker.patch("swh.loader.package.pypi.loader.PyPILoader.load") mock_load.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.package.pypi.tasks.LoadPyPI", kwargs=dict(url="some-url") ) assert res res.wait() assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"} + + +def test_tasks_pypi_loader_for_listed_origin( + mocker, + swh_scheduler_celery_app, + pypi_lister, + pypi_listed_origin, +): + mock_load = mocker.patch("swh.loader.package.pypi.loader.PyPILoader.load") + mock_load.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(pypi_listed_origin, pypi_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.pypi.tasks.LoadPyPI", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"}