diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,4 +1,4 @@ swh.core >= 0.0.75 swh.model >= 0.3.0 swh.scheduler -swh.storage >= 0.2.0 +swh.storage >= 0.3.0 diff --git a/swh/loader/cli.py b/swh/loader/cli.py --- a/swh/loader/cli.py +++ b/swh/loader/cli.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -7,6 +7,7 @@ import logging import click +import iso8601 import pkg_resources from typing import Any @@ -70,6 +71,11 @@ """Ingest with loader the origin located at """ (_, kw) = parse_options(options) logger.debug(f"kw: {kw}") + visit_date = kw.get("visit_date") + if visit_date and isinstance(visit_date, str): + visit_date = iso8601.parse_date(visit_date) + kw["visit_date"] = visit_date + loader = get_loader(type, url=url, **kw) result = loader.load() click.echo(result) diff --git a/swh/loader/core/loader.py b/swh/loader/core/loader.py --- a/swh/loader/core/loader.py +++ b/swh/loader/core/loader.py @@ -9,7 +9,7 @@ import os from abc import ABCMeta, abstractmethod -from typing import Any, Dict, Iterable, Optional, Tuple, Union +from typing import Any, Dict, Iterable, Optional, Tuple from swh.core import config from swh.model.model import ( @@ -18,6 +18,7 @@ SkippedContent, Directory, Origin, + OriginVisit, OriginVisitStatus, Revision, Release, @@ -98,7 +99,7 @@ self.max_content_size = self.config["max_content_size"] # possibly overridden in self.prepare method - self.visit_date: Optional[Union[str, datetime.datetime]] = None + self.visit_date: Optional[datetime.datetime] = None self.origin: Optional[Origin] = None @@ -173,9 +174,19 @@ if not self.visit_date: # now as default visit_date if not provided self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) + assert isinstance(self.visit_date, datetime.datetime) + assert isinstance(self.visit_type, str) self.visit = self.storage.origin_visit_add( - self.origin.url, self.visit_date, self.visit_type - ) + [ + OriginVisit( + origin=self.origin.url, + date=self.visit_date, + type=self.visit_type, + status="ongoing", + snapshot=None, + ) + ] + )[0] @abstractmethod def prepare(self, *args, **kwargs) -> None: diff --git a/swh/loader/core/tests/test_loader.py b/swh/loader/core/tests/test_loader.py --- a/swh/loader/core/tests/test_loader.py +++ b/swh/loader/core/tests/test_loader.py @@ -7,7 +7,7 @@ import hashlib import logging -from swh.model.model import Origin, Snapshot +from swh.model.model import Origin, OriginVisit, Snapshot from swh.loader.core.loader import BaseLoader, DVCSLoader @@ -33,9 +33,14 @@ self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) self.visit_type = "git" origin_url = self.storage.origin_add_one(ORIGIN) - self.visit = self.storage.origin_visit_add( - origin_url, self.visit_date, self.visit_type + visit = OriginVisit( + origin=origin_url, + date=self.visit_date, + type=self.visit_type, + status="ongoing", + snapshot=None, ) + self.visit = self.storage.origin_visit_add([visit])[0] class DummyDVCSLoader(DummyLoader, DVCSLoader): diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py --- a/swh/loader/package/loader.py +++ b/swh/loader/package/loader.py @@ -27,6 +27,7 @@ TargetType, Snapshot, Origin, + OriginVisit, OriginVisitStatus, ) from swh.storage import get_storage @@ -286,6 +287,7 @@ snapshot_id: Optional[bytes] = None if snapshot and snapshot.id: # to prevent the snapshot.id to b"" snapshot_id = snapshot.id + assert visit.visit visit_status = OriginVisitStatus( origin=self.url, visit=visit.visit, @@ -306,8 +308,16 @@ try: self.storage.origin_add_one(origin) visit = self.storage.origin_visit_add( - self.url, date=self.visit_date, type=self.visit_type - ) + [ + OriginVisit( + origin=self.url, + date=self.visit_date, + type=self.visit_type, + status="ongoing", + snapshot=None, + ) + ] + )[0] except Exception as e: logger.exception("Failed to initialize origin_visit for %s", self.url) sentry_sdk.capture_exception(e) diff --git a/swh/loader/package/tests/test_loader.py b/swh/loader/package/tests/test_loader.py --- a/swh/loader/package/tests/test_loader.py +++ b/swh/loader/package/tests/test_loader.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -15,7 +15,7 @@ def origin_add_one(self, origin): return origin - def origin_visit_add(self, origin, date, type): + def origin_visit_add(self, visits): raise ValueError("We refuse to add an origin visit") diff --git a/swh/loader/tests/test_cli.py b/swh/loader/tests/test_cli.py --- a/swh/loader/tests/test_cli.py +++ b/swh/loader/tests/test_cli.py @@ -1,8 +1,10 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import datetime + import pytest @@ -77,6 +79,24 @@ mock_loader.assert_called_once_with(url="https://some-url") # constructor +def test_run_with_visit_date(mocker, swh_config): + """iso visit_date parameter should be parsed as datetime + + """ + mock_loader = mocker.patch("swh.loader.cli.get_loader") + + runner = CliRunner() + input_date = "2016-05-03 15:16:32+00" + result = runner.invoke(run, ["npm", "https://some-url", f"visit_date={input_date}"]) + assert result.exit_code == 0 + expected_parsed_date = datetime.datetime( + 2016, 5, 3, 15, 16, 32, tzinfo=datetime.timezone.utc + ) + mock_loader.assert_called_once_with( + "npm", url="https://some-url", visit_date=expected_parsed_date + ) + + def test_list_help(mocker, swh_config): """Triggering a load should be ok