diff --git a/swh/loader/git/tasks.py b/swh/loader/git/tasks.py --- a/swh/loader/git/tasks.py +++ b/swh/loader/git/tasks.py @@ -5,51 +5,39 @@ import dateutil.parser -from swh.scheduler.task import Task +from celery import current_app as app -from .from_disk import GitLoaderFromDisk, GitLoaderFromArchive -from .loader import GitLoader +from swh.loader.git.from_disk import GitLoaderFromDisk, GitLoaderFromArchive +from swh.loader.git.loader import GitLoader -# TODO: rename to LoadRemoteGitRepository -class UpdateGitRepository(Task): +@app.task(name=__name__ + '.UpdateGitRepository') +def update_git_repository(repo_url, base_url=None): """Import a git repository from a remote location""" - task_queue = 'swh_loader_git' + loader = GitLoader() + return loader.load(repo_url, base_url=base_url) - def run_task(self, repo_url, base_url=None): - """Import a git repository""" - loader = GitLoader() - loader.log = self.log - return loader.load(repo_url, base_url=base_url) +@app.task(name=__name__ + '.LoadDiskGitRepository') +def load_disk_git_repository(origin_url, directory, date): + """Import a git repository from disk + Import a git repository, cloned in `directory` from `origin_url` at + `date`. -class LoadDiskGitRepository(Task): - """Import a git repository from disk""" - task_queue = 'swh_loader_git_express' + """ + loader = GitLoaderFromDisk() + return loader.load(origin_url, directory, dateutil.parser.parse(date)) - def run_task(self, origin_url, directory, date): - """Import a git repository, cloned in `directory` from `origin_url` at - `date`.""" - loader = GitLoaderFromDisk() - loader.log = self.log +@app.task(name=__name__ + '.UncompressAndLoadDiskGitRepository') +def run_task(origin_url, archive_path, date): + """Import a git repository from a zip archive - return loader.load(origin_url, directory, dateutil.parser.parse(date)) - - -class UncompressAndLoadDiskGitRepository(Task): - """Import a git repository from a zip archive""" - task_queue = 'swh_loader_git_archive' - - def run_task(self, origin_url, archive_path, date): - """1. Uncompress an archive repository in a local and temporary folder - 2. Load it through the git disk loader - 3. Clean up the temporary folder - - """ - loader = GitLoaderFromArchive() - loader.log = self.log - - return loader.load( - origin_url, archive_path, dateutil.parser.parse(date)) + 1. Uncompress an archive repository in a local and temporary folder + 2. Load it through the git disk loader + 3. Clean up the temporary folder + """ + loader = GitLoaderFromArchive() + return loader.load( + origin_url, archive_path, dateutil.parser.parse(date)) diff --git a/swh/loader/git/tests/conftest.py b/swh/loader/git/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/loader/git/tests/conftest.py @@ -0,0 +1,10 @@ +import pytest + +from swh.scheduler.tests.conftest import * # noqa + + +@pytest.fixture(scope='session') +def celery_includes(): + return [ + 'swh.loader.git.tasks', + ] diff --git a/swh/loader/git/tests/test_tasks.py b/swh/loader/git/tests/test_tasks.py --- a/swh/loader/git/tests/test_tasks.py +++ b/swh/loader/git/tests/test_tasks.py @@ -4,68 +4,51 @@ # See top-level LICENSE file for more information import datetime -import unittest from unittest.mock import patch -from swh.loader.git.tasks import ( - UpdateGitRepository, LoadDiskGitRepository, - UncompressAndLoadDiskGitRepository -) +@patch('swh.loader.git.loader.GitLoader.load') +def test_git_loader(mock_loader, swh_app, celery_session_worker): + mock_loader.return_value = {'status': 'eventful'} -class TestTasks(unittest.TestCase): - def test_check_task_name(self): - task = UpdateGitRepository() - self.assertEqual(task.task_queue, 'swh_loader_git') + res = swh_app.send_task( + 'swh.loader.git.tasks.UpdateGitRepository', + ('origin_url',)) + assert res + res.wait() + assert res.successful() - @patch('swh.loader.git.loader.GitLoader.load') - def test_task(self, mock_loader): - mock_loader.return_value = {'status': 'eventful'} - task = UpdateGitRepository() + assert res.result == {'status': 'eventful'} + mock_loader.assert_called_once_with('origin_url', base_url=None) - # given - actual_result = task.run_task('origin_url') - self.assertEqual(actual_result, {'status': 'eventful'}) +@patch('swh.loader.git.from_disk.GitLoaderFromDisk.load') +def test_git_loader_from_disk(mock_loader, swh_app, celery_session_worker): + mock_loader.return_value = {'status': 'uneventful'} - mock_loader.assert_called_once_with('origin_url', base_url=None) + res = swh_app.send_task( + 'swh.loader.git.tasks.LoadDiskGitRepository', + ('origin_url2', '/some/repo', '2018-12-10 00:00')) + assert res + res.wait() + assert res.successful() + assert res.result == {'status': 'uneventful'} + mock_loader.assert_called_once_with( + 'origin_url2', '/some/repo', datetime.datetime(2018, 12, 10, 0, 0)) -class TestTasks2(unittest.TestCase): - def test_check_task_name(self): - task = LoadDiskGitRepository() - self.assertEqual(task.task_queue, 'swh_loader_git_express') - @patch('swh.loader.git.from_disk.GitLoaderFromDisk.load') - def test_task(self, mock_loader): - mock_loader.return_value = {'status': 'uneventful'} - task = LoadDiskGitRepository() +@patch('swh.loader.git.from_disk.GitLoaderFromArchive.load') +def test_git_loader_from_archive(mock_loader, swh_app, celery_session_worker): + mock_loader.return_value = {'status': 'failed'} - # given - actual_result = task.run_task('origin_url2', '/some/repo', - '2018-12-10 00:00') + res = swh_app.send_task( + 'swh.loader.git.tasks.UncompressAndLoadDiskGitRepository', + ('origin_url3', '/some/repo', '2017-01-10 00:00')) + assert res + res.wait() + assert res.successful() - self.assertEqual(actual_result, {'status': 'uneventful'}) - - mock_loader.assert_called_once_with( - 'origin_url2', '/some/repo', datetime.datetime(2018, 12, 10, 0, 0)) - - -class TestTasks3(unittest.TestCase): - def test_check_task_name(self): - task = UncompressAndLoadDiskGitRepository() - self.assertEqual(task.task_queue, 'swh_loader_git_archive') - - @patch('swh.loader.git.from_disk.GitLoaderFromArchive.load') - def test_task(self, mock_loader): - mock_loader.return_value = {'status': 'failed'} - task = UncompressAndLoadDiskGitRepository() - - # given - actual_result = task.run_task('origin_url3', '/path/repo', - '2017-01-10 00:00') - - self.assertEqual(actual_result, {'status': 'failed'}) - - mock_loader.assert_called_once_with( - 'origin_url3', '/path/repo', datetime.datetime(2017, 1, 10, 0, 0)) + assert res.result == {'status': 'failed'} + mock_loader.assert_called_once_with( + 'origin_url3', '/some/repo', datetime.datetime(2017, 1, 10, 0, 0))