Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9341037
D5024.id17957.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
6 KB
Subscribers
None
D5024.id17957.diff
View Options
diff --git a/swh/loader/package/cran/tests/test_cran.py b/swh/loader/package/cran/tests/test_cran.py
--- a/swh/loader/package/cran/tests/test_cran.py
+++ b/swh/loader/package/cran/tests/test_cran.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2020 The Software Heritage developers
+# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -6,6 +6,7 @@
from datetime import datetime, timezone
import os
from os import path
+from unittest.mock import patch
from dateutil.tz import tzlocal
import pytest
@@ -309,3 +310,53 @@
"Repository": "CRAN",
"Date/Publication": "2019-01-31 20:53:50 UTC",
}
+
+
+@pytest.mark.parametrize(
+ "method_name",
+ ["build_extrinsic_snapshot_metadata", "build_extrinsic_origin_metadata",],
+)
+def test_cran_fail_to_build_or_load_extrinsic_metadata(
+ method_name, swh_config, requests_mock_datadir
+):
+ """problem during loading: {visit: failed, status: failed, no snapshot}
+
+ """
+ version = "2.22-6"
+ base_url = "https://cran.r-project.org"
+ origin_url = f"{base_url}/Packages/Recommended_KernSmooth/index.html"
+ artifact_url = (
+ f"{base_url}/src_contrib_1.4.0_Recommended_KernSmooth_{version}.tar.gz" # noqa
+ )
+
+ full_method_name = f"swh.loader.package.cran.loader.CRANLoader.{method_name}"
+ with patch(
+ full_method_name,
+ side_effect=ValueError("Fake to fail to build or load extrinsic metadata"),
+ ):
+ loader = CRANLoader(
+ origin_url, artifacts=[{"url": artifact_url, "version": version}]
+ )
+
+ actual_load_status = loader.load()
+
+ assert actual_load_status == {
+ "status": "failed",
+ "snapshot_id": SNAPSHOT.id.hex(),
+ }
+
+ visit_stats = get_stats(loader.storage)
+ assert {
+ "content": 33,
+ "directory": 7,
+ "origin": 1,
+ "origin_visit": 1,
+ "release": 0,
+ "revision": 1,
+ "skipped_content": 0,
+ "snapshot": 1,
+ } == visit_stats
+
+ assert_last_visit_matches(
+ loader.storage, origin_url, status="partial", type="cran"
+ )
diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py
--- a/swh/loader/package/loader.py
+++ b/swh/loader/package/loader.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2020 The Software Heritage developers
+# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -340,7 +340,7 @@
"""
status_load = "uneventful" # either: eventful, uneventful, failed
- status_visit = "full" # either: partial, full
+ status_visit = "full" # see swh.model.model.OriginVisitStatus
tmp_revisions = {} # type: Dict[str, List]
snapshot = None
failed_branches: List[str] = []
@@ -407,7 +407,7 @@
except Exception as e:
logger.exception("Failed to get previous state for %s", self.url)
sentry_sdk.capture_exception(e)
- status_visit = "partial"
+ status_visit = "failed"
status_load = "failed"
return finalize_visit()
@@ -452,7 +452,7 @@
if not tmp_revisions:
# We could not load any revisions; fail completely
- status_visit = "partial"
+ status_visit = "failed"
status_load = "failed"
return finalize_visit()
@@ -471,7 +471,7 @@
except Exception as e:
logger.exception("Failed to build snapshot for origin %s", self.url)
sentry_sdk.capture_exception(e)
- status_visit = "partial"
+ status_visit = "failed"
status_load = "failed"
if snapshot:
diff --git a/swh/loader/package/npm/tests/test_npm.py b/swh/loader/package/npm/tests/test_npm.py
--- a/swh/loader/package/npm/tests/test_npm.py
+++ b/swh/loader/package/npm/tests/test_npm.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2020 The Software Heritage developers
+# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -700,4 +700,4 @@
"status": "failed",
}
- assert_last_visit_matches(loader.storage, url, status="partial", type="npm")
+ assert_last_visit_matches(loader.storage, url, status="failed", type="npm")
diff --git a/swh/loader/package/pypi/tests/test_pypi.py b/swh/loader/package/pypi/tests/test_pypi.py
--- a/swh/loader/package/pypi/tests/test_pypi.py
+++ b/swh/loader/package/pypi/tests/test_pypi.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2020 The Software Heritage developers
+# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -251,6 +251,36 @@
assert_last_visit_matches(loader.storage, url, status="partial", type="pypi")
+def test_pypi_fail__load_snapshot(swh_config, requests_mock_datadir):
+ """problem during loading: {visit: failed, status: failed, no snapshot}
+
+ """
+ url = "https://pypi.org/project/0805nexter"
+ with patch(
+ "swh.loader.package.pypi.loader.PyPILoader._load_snapshot",
+ side_effect=ValueError("Fake problem to fail visit"),
+ ):
+ loader = PyPILoader(url)
+
+ actual_load_status = loader.load()
+ assert actual_load_status == {"status": "failed"}
+
+ stats = get_stats(loader.storage)
+
+ assert {
+ "content": 6,
+ "directory": 4,
+ "origin": 1,
+ "origin_visit": 1,
+ "release": 0,
+ "revision": 2,
+ "skipped_content": 0,
+ "snapshot": 0,
+ } == stats
+
+ assert_last_visit_matches(loader.storage, url, status="failed", type="pypi")
+
+
# problem during loading:
# {visit: partial, status: uneventful, no snapshot}
@@ -279,7 +309,7 @@
"snapshot": 0,
} == stats
- assert_last_visit_matches(loader.storage, url, status="partial", type="pypi")
+ assert_last_visit_matches(loader.storage, url, status="failed", type="pypi")
# problem during loading: failure early enough in between swh contents...
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Thu, Jul 3, 11:33 AM (3 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3221117
Attached To
D5024: package: Mark visit status as failed when relevant
Event Timeline
Log In to Comment