Page MenuHomeSoftware Heritage
Files F9340992

No OneTemporary
Actions

View Options

diff --git a/requirements-test.txt b/requirements-test.txt
index 78f8c9b..024f1a4 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -1,7 +1,6 @@
pytest
pytest-mock
swh.scheduler[testing] >= 0.5.0
swh.storage[testing]
types-click
types-Deprecated
-types-python-dateutil
diff --git a/swh/loader/git/tasks.py b/swh/loader/git/tasks.py
index 51fe046..6a0f136 100644
--- a/swh/loader/git/tasks.py
+++ b/swh/loader/git/tasks.py
@@ -1,50 +1,45 @@
-# Copyright (C) 2015-2021 The Software Heritage developers
+# Copyright (C) 2015-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from typing import Any, Dict, Optional
+from typing import Any, Dict
from celery import shared_task
-import dateutil.parser
+from swh.loader.core.utils import parse_visit_date
from swh.loader.git.from_disk import GitLoaderFromArchive, GitLoaderFromDisk
from swh.loader.git.loader import GitLoader
+def _process_kwargs(kwargs):
+ if "visit_date" in kwargs:
+ kwargs["visit_date"] = parse_visit_date(kwargs["visit_date"])
+ return kwargs
+
+
@shared_task(name=__name__ + ".UpdateGitRepository")
-def load_git(*, url: str, base_url: Optional[str] = None) -> Dict[str, Any]:
+def load_git(**kwargs) -> Dict[str, Any]:
"""Import a git repository from a remote location"""
- loader = GitLoader.from_configfile(url=url, base_url=base_url)
+ loader = GitLoader.from_configfile(**_process_kwargs(kwargs))
return loader.load()
@shared_task(name=__name__ + ".LoadDiskGitRepository")
-def load_git_from_dir(*, url: str, directory: str, date: str) -> Dict[str, Any]:
- """Import a git repository from a local repository
-
- Import a git repository, cloned in `directory` from `origin_url` at
- `date`.
-
- """
- visit_date = dateutil.parser.parse(date)
- loader = GitLoaderFromDisk.from_configfile(
- url=url, directory=directory, visit_date=visit_date
- )
+def load_git_from_dir(**kwargs) -> Dict[str, Any]:
+ """Import a git repository from a local repository"""
+ loader = GitLoaderFromDisk.from_configfile(**_process_kwargs(kwargs))
return loader.load()
@shared_task(name=__name__ + ".UncompressAndLoadDiskGitRepository")
-def load_git_from_zip(*, url: str, archive_path: str, date: str) -> Dict[str, Any]:
+def load_git_from_zip(**kwargs) -> Dict[str, Any]:
"""Import a git repository from a zip archive
1. Uncompress an archive repository in a local and temporary folder
2. Load it through the git disk loader
3. Clean up the temporary folder
"""
- visit_date = dateutil.parser.parse(date)
- loader = GitLoaderFromArchive.from_configfile(
- url=url, archive_path=archive_path, visit_date=visit_date
- )
+ loader = GitLoaderFromArchive.from_configfile(**_process_kwargs(kwargs))
return loader.load()
diff --git a/swh/loader/git/tests/test_tasks.py b/swh/loader/git/tests/test_tasks.py
index 0ec2596..fd442e1 100644
--- a/swh/loader/git/tests/test_tasks.py
+++ b/swh/loader/git/tests/test_tasks.py
@@ -1,69 +1,164 @@
-# Copyright (C) 2018-2020 The Software Heritage developers
+# Copyright (C) 2018-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import uuid
+
+import pytest
+
+from swh.scheduler.model import ListedOrigin, Lister
+from swh.scheduler.utils import create_origin_task_dict
+
+
+@pytest.fixture(autouse=True)
+def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config):
+ pass
+
+
+@pytest.fixture
+def git_lister():
+ return Lister(name="git-lister", instance_name="example", id=uuid.uuid4())
+
+
+@pytest.fixture
+def git_listed_origin(git_lister):
+ return ListedOrigin(
+ lister_id=git_lister.id, url="https://git.example.org/repo", visit_type="git"
+ )
+
def test_git_loader(
- mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker
+ mocker,
+ swh_scheduler_celery_app,
):
mock_loader = mocker.patch("swh.loader.git.loader.GitLoader.load")
mock_loader.return_value = {"status": "eventful"}
res = swh_scheduler_celery_app.send_task(
"swh.loader.git.tasks.UpdateGitRepository",
- kwargs={
- "url": "origin_url",
- },
+ kwargs={"url": "origin_url"},
+ )
+ assert res
+ res.wait()
+ assert res.successful()
+
+ assert res.result == {"status": "eventful"}
+ mock_loader.assert_called_once_with()
+
+
+def test_git_loader_for_listed_origin(
+ mocker,
+ swh_scheduler_celery_app,
+ git_lister,
+ git_listed_origin,
+):
+ mock_loader = mocker.patch("swh.loader.git.loader.GitLoader.load")
+ mock_loader.return_value = {"status": "eventful"}
+
+ task_dict = create_origin_task_dict(git_listed_origin, git_lister)
+
+ res = swh_scheduler_celery_app.send_task(
+ "swh.loader.git.tasks.UpdateGitRepository",
+ kwargs=task_dict["arguments"]["kwargs"],
)
assert res
res.wait()
assert res.successful()
assert res.result == {"status": "eventful"}
mock_loader.assert_called_once_with()
def test_git_loader_from_disk(
- mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker
+ mocker,
+ swh_scheduler_celery_app,
):
mock_loader = mocker.patch("swh.loader.git.from_disk.GitLoaderFromDisk.load")
mock_loader.return_value = {"status": "uneventful"}
res = swh_scheduler_celery_app.send_task(
"swh.loader.git.tasks.LoadDiskGitRepository",
- kwargs={
- "url": "origin_url2",
- "directory": "/some/repo",
- "date": "2018-12-10 00:00",
- },
+ kwargs={"url": "origin_url2", "directory": "/some/repo", "visit_date": "now"},
+ )
+ assert res
+ res.wait()
+ assert res.successful()
+
+ assert res.result == {"status": "uneventful"}
+ mock_loader.assert_called_once_with()
+
+
+def test_git_loader_from_disk_for_listed_origin(
+ mocker,
+ swh_scheduler_celery_app,
+ git_lister,
+ git_listed_origin,
+):
+ mock_loader = mocker.patch("swh.loader.git.from_disk.GitLoaderFromDisk.load")
+ mock_loader.return_value = {"status": "uneventful"}
+
+ git_listed_origin.extra_loader_arguments = {
+ "directory": "/some/repo",
+ }
+ task_dict = create_origin_task_dict(git_listed_origin, git_lister)
+
+ res = swh_scheduler_celery_app.send_task(
+ "swh.loader.git.tasks.LoadDiskGitRepository",
+ kwargs=task_dict["arguments"]["kwargs"],
)
assert res
res.wait()
assert res.successful()
assert res.result == {"status": "uneventful"}
mock_loader.assert_called_once_with()
def test_git_loader_from_archive(
- mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker
+ mocker,
+ swh_scheduler_celery_app,
):
mock_loader = mocker.patch("swh.loader.git.from_disk.GitLoaderFromArchive.load")
-
mock_loader.return_value = {"status": "failed"}
res = swh_scheduler_celery_app.send_task(
"swh.loader.git.tasks.UncompressAndLoadDiskGitRepository",
kwargs={
"url": "origin_url3",
"archive_path": "/some/repo",
- "date": "2017-01-10 00:00",
+ "visit_date": "now",
},
)
assert res
res.wait()
assert res.successful()
assert res.result == {"status": "failed"}
mock_loader.assert_called_once_with()
+
+
+def test_git_loader_from_archive_for_listed_origin(
+ mocker,
+ swh_scheduler_celery_app,
+ git_lister,
+ git_listed_origin,
+):
+ mock_loader = mocker.patch("swh.loader.git.from_disk.GitLoaderFromArchive.load")
+ mock_loader.return_value = {"status": "failed"}
+
+ git_listed_origin.extra_loader_arguments = {
+ "archive_path": "/some/repo",
+ }
+ task_dict = create_origin_task_dict(git_listed_origin, git_lister)
+
+ res = swh_scheduler_celery_app.send_task(
+ "swh.loader.git.tasks.UncompressAndLoadDiskGitRepository",
+ kwargs=task_dict["arguments"]["kwargs"],
+ )
+ assert res
+ res.wait()
+ assert res.successful()
+
+ assert res.result == {"status": "failed"}
+ mock_loader.assert_called_once_with()

File Metadata

Mime Type
text/x-diff
Expires
Fri, Jul 4, 11:28 AM (3 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3259303

Event Timeline

About · Developer information · Legal