diff --git a/requirements-test.txt b/requirements-test.txt index e079f8a..584dbb5 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1,2 @@ pytest +swh.scheduler[testing] diff --git a/swh/loader/cvs/tasks.py b/swh/loader/cvs/tasks.py index 63867dc..a3753ba 100644 --- a/swh/loader/cvs/tasks.py +++ b/swh/loader/cvs/tasks.py @@ -1,54 +1,40 @@ # Copyright (C) 2015-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime from typing import Optional from celery import shared_task import iso8601 from .loader import CvsLoader def convert_to_datetime(date: Optional[str]) -> Optional[datetime]: if date is None: return None try: assert isinstance(date, str) return iso8601.parse_date(date) except Exception: return None @shared_task(name=__name__ + ".LoadCvsRepository") def load_cvs( - *, - url: str, - origin_url: Optional[str] = None, - destination_path: Optional[str] = None, - swh_revision: Optional[str] = None, - visit_date: Optional[str] = None, + *, url: str, origin_url: Optional[str] = None, visit_date: Optional[str] = None, ): """Import a CVS repository Args: - url: (mandatory) CVS's repository url to ingest data from - origin_url: Optional original url override to use as origin reference in the archive. If not provided, "url" is used as origin. - - destination_path: (optional) root directory to - locally retrieve svn's data - - swh_revision: (optional) extra revision hex to - start from. See swh.loader.svn.CvsLoader.process - docstring - visit_date: Optional date to override the visit date """ loader = CvsLoader.from_configfile( - url=url, - origin_url=origin_url, - destination_path=destination_path, - swh_revision=swh_revision, - visit_date=convert_to_datetime(visit_date), + url=url, origin_url=origin_url, visit_date=convert_to_datetime(visit_date), ) return loader.load() diff --git a/swh/loader/cvs/tests/test_tasks.py b/swh/loader/cvs/tests/test_tasks.py index 332c43e..126fd3c 100644 --- a/swh/loader/cvs/tests/test_tasks.py +++ b/swh/loader/cvs/tests/test_tasks.py @@ -1,25 +1,43 @@ # Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime, timezone import pytest from swh.loader.cvs.tasks import convert_to_datetime @pytest.mark.parametrize( "date,expected_result", [ (None, None), ( "2021-11-23 09:41:02.434195+00:00", datetime(2021, 11, 23, 9, 41, 2, 434195, tzinfo=timezone.utc), ), ("23112021", None,), # failure to parse ], ) def test_convert_to_datetime(date, expected_result): assert convert_to_datetime(date) == expected_result + + +def test_cvs_loader( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + mock_loader = mocker.patch("swh.loader.cvs.loader.CvsLoader.load") + mock_loader.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.loader.cvs.tasks.LoadCvsRepository", + kwargs=dict(url="some-technical-url", origin_url="origin-url"), + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + assert mock_loader.called diff --git a/tox.ini b/tox.ini index 010ee68..a218910 100644 --- a/tox.ini +++ b/tox.ini @@ -1,73 +1,77 @@ [tox] envlist=black,flake8,mypy,py3 [testenv] extras = testing deps = pytest-cov + # the dependency below is needed for now as a workaround for + # https://github.com/pypa/pip/issues/6239 + # TODO: remove when this issue is fixed + swh.scheduler[testing] commands = pytest --doctest-modules \ {envsitepackagesdir}/swh/loader/cvs \ --cov={envsitepackagesdir}/swh/loader/cvs \ --cov-branch {posargs} [testenv:black] skip_install = true deps = black==19.10b0 commands = {envpython} -m black --check swh [testenv:flake8] skip_install = true deps = flake8 commands = {envpython} -m flake8 [testenv:mypy] extras = testing deps = - mypy + mypy!=0.920 commands = mypy swh # build documentation outside swh-environment using the current # git HEAD of swh-docs, is executed on CI for each diff to prevent # breaking doc build [testenv:sphinx] whitelist_externals = make usedevelop = true extras = testing deps = # fetch and install swh-docs in develop mode -e git+https://forge.softwareheritage.org/source/swh-docs#egg=swh.docs setenv = SWH_PACKAGE_DOC_TOX_BUILD = 1 # turn warnings into errors SPHINXOPTS = -W commands = make -I ../.tox/sphinx/src/swh-docs/swh/ -C docs # build documentation only inside swh-environment using local state # of swh-docs package [testenv:sphinx-dev] whitelist_externals = make usedevelop = true extras = testing deps = # install swh-docs in develop mode -e ../swh-docs setenv = SWH_PACKAGE_DOC_TOX_BUILD = 1 # turn warnings into errors SPHINXOPTS = -W commands = make -I ../.tox/sphinx-dev/src/swh-docs/swh/ -C docs