diff --git a/swh/loader/package/cran/tests/test_cran.py b/swh/loader/package/cran/tests/test_cran.py --- a/swh/loader/package/cran/tests/test_cran.py +++ b/swh/loader/package/cran/tests/test_cran.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2020 The Software Heritage developers +# Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -6,6 +6,7 @@ from datetime import datetime, timezone import os from os import path +from unittest.mock import patch from dateutil.tz import tzlocal import pytest @@ -309,3 +310,95 @@ "Repository": "CRAN", "Date/Publication": "2019-01-31 20:53:50 UTC", } + + +def test_cran_fail_build_extrinsic_snapshot_metadata(swh_config, requests_mock_datadir): + """problem during loading: {visit: failed, status: failed, no snapshot} + + """ + version = "2.22-6" + base_url = "https://cran.r-project.org" + origin_url = f"{base_url}/Packages/Recommended_KernSmooth/index.html" + artifact_url = ( + f"{base_url}/src_contrib_1.4.0_Recommended_KernSmooth_{version}.tar.gz" # noqa + ) + with patch( + "swh.loader.package.cran.loader.CRANLoader.build_extrinsic_origin_metadata", + side_effect=ValueError("Fake to build extrinsic metadata"), + ): + loader = CRANLoader( + origin_url, artifacts=[{"url": artifact_url, "version": version}] + ) + + actual_load_status = loader.load() + + assert actual_load_status == { + "status": "failed", + "snapshot_id": SNAPSHOT.id.hex(), + } + + visit_stats = get_stats(loader.storage) + assert { + "content": 33, + "directory": 7, + "origin": 1, + "origin_visit": 1, + "release": 0, + "revision": 1, + "skipped_content": 0, + "snapshot": 1, + } == visit_stats + + assert_last_visit_matches( + loader.storage, origin_url, status="failed", type="cran" + ) + + +@pytest.mark.parametrize( + "method_name", + ["build_extrinsic_snapshot_metadata", "build_extrinsic_origin_metadata",], +) +def test_cran_fail_to_build_or_load_extrinsic_metadata( + method_name, swh_config, requests_mock_datadir +): + """problem during loading: {visit: failed, status: failed, no snapshot} + + """ + version = "2.22-6" + base_url = "https://cran.r-project.org" + origin_url = f"{base_url}/Packages/Recommended_KernSmooth/index.html" + artifact_url = ( + f"{base_url}/src_contrib_1.4.0_Recommended_KernSmooth_{version}.tar.gz" # noqa + ) + + full_method_name = f"swh.loader.package.cran.loader.CRANLoader.{method_name}" + with patch( + full_method_name, + side_effect=ValueError("Fake to fail to build or load extrinsic metadata"), + ): + loader = CRANLoader( + origin_url, artifacts=[{"url": artifact_url, "version": version}] + ) + + actual_load_status = loader.load() + + assert actual_load_status == { + "status": "failed", + "snapshot_id": SNAPSHOT.id.hex(), + } + + visit_stats = get_stats(loader.storage) + assert { + "content": 33, + "directory": 7, + "origin": 1, + "origin_visit": 1, + "release": 0, + "revision": 1, + "skipped_content": 0, + "snapshot": 1, + } == visit_stats + + assert_last_visit_matches( + loader.storage, origin_url, status="failed", type="cran" + ) diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py --- a/swh/loader/package/loader.py +++ b/swh/loader/package/loader.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2020 The Software Heritage developers +# Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -340,7 +340,7 @@ """ status_load = "uneventful" # either: eventful, uneventful, failed - status_visit = "full" # either: partial, full + status_visit = "full" # see swh.model.model.OriginVisitStatus tmp_revisions = {} # type: Dict[str, List] snapshot = None failed_branches: List[str] = [] @@ -407,7 +407,7 @@ except Exception as e: logger.exception("Failed to get previous state for %s", self.url) sentry_sdk.capture_exception(e) - status_visit = "partial" + status_visit = "failed" status_load = "failed" return finalize_visit() @@ -452,7 +452,7 @@ if not tmp_revisions: # We could not load any revisions; fail completely - status_visit = "partial" + status_visit = "failed" status_load = "failed" return finalize_visit() @@ -471,7 +471,7 @@ except Exception as e: logger.exception("Failed to build snapshot for origin %s", self.url) sentry_sdk.capture_exception(e) - status_visit = "partial" + status_visit = "failed" status_load = "failed" if snapshot: @@ -483,7 +483,7 @@ "Failed to load extrinsic snapshot metadata for %s", self.url ) sentry_sdk.capture_exception(e) - status_visit = "partial" + status_visit = "failed" status_load = "failed" try: @@ -494,7 +494,7 @@ "Failed to load extrinsic origin metadata for %s", self.url ) sentry_sdk.capture_exception(e) - status_visit = "partial" + status_visit = "failed" status_load = "failed" return finalize_visit() diff --git a/swh/loader/package/npm/tests/test_npm.py b/swh/loader/package/npm/tests/test_npm.py --- a/swh/loader/package/npm/tests/test_npm.py +++ b/swh/loader/package/npm/tests/test_npm.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2020 The Software Heritage developers +# Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -700,4 +700,4 @@ "status": "failed", } - assert_last_visit_matches(loader.storage, url, status="partial", type="npm") + assert_last_visit_matches(loader.storage, url, status="failed", type="npm") diff --git a/swh/loader/package/pypi/tests/test_pypi.py b/swh/loader/package/pypi/tests/test_pypi.py --- a/swh/loader/package/pypi/tests/test_pypi.py +++ b/swh/loader/package/pypi/tests/test_pypi.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2020 The Software Heritage developers +# Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -251,6 +251,36 @@ assert_last_visit_matches(loader.storage, url, status="partial", type="pypi") +def test_pypi_fail__load_snapshot(swh_config, requests_mock_datadir): + """problem during loading: {visit: failed, status: failed, no snapshot} + + """ + url = "https://pypi.org/project/0805nexter" + with patch( + "swh.loader.package.pypi.loader.PyPILoader._load_snapshot", + side_effect=ValueError("Fake problem to fail visit"), + ): + loader = PyPILoader(url) + + actual_load_status = loader.load() + assert actual_load_status == {"status": "failed"} + + stats = get_stats(loader.storage) + + assert { + "content": 6, + "directory": 4, + "origin": 1, + "origin_visit": 1, + "release": 0, + "revision": 2, + "skipped_content": 0, + "snapshot": 0, + } == stats + + assert_last_visit_matches(loader.storage, url, status="failed", type="pypi") + + # problem during loading: # {visit: partial, status: uneventful, no snapshot} @@ -279,7 +309,7 @@ "snapshot": 0, } == stats - assert_last_visit_matches(loader.storage, url, status="partial", type="pypi") + assert_last_visit_matches(loader.storage, url, status="failed", type="pypi") # problem during loading: failure early enough in between swh contents... diff --git a/swh/loader/package/tests/test_loader.py b/swh/loader/package/tests/test_loader.py --- a/swh/loader/package/tests/test_loader.py +++ b/swh/loader/package/tests/test_loader.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2020 The Software Heritage developers +# Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -16,6 +16,9 @@ def origin_visit_get_latest(self, origin): return None + def flush(self): + pass + class FakeStorage2(FakeStorage): def origin_add(self, origins): @@ -24,6 +27,9 @@ def origin_visit_add(self, visits): raise ValueError("We refuse to add an origin visit") + def flush(self): + pass + def test_loader_origin_visit_failure(swh_config): """Failure to add origin or origin visit should failed immediately