Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/pypi/tests/test_pypi.py
# Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import os | import os | ||||
from os import path | from os import path | ||||
import pytest | import pytest | ||||
from unittest.mock import patch | from unittest.mock import patch | ||||
from swh.core.tarball import uncompress | from swh.core.tarball import uncompress | ||||
from swh.core.pytest_plugin import requests_mock_datadir_factory | from swh.core.pytest_plugin import requests_mock_datadir_factory | ||||
from swh.model.hashutil import hash_to_bytes | from swh.model.hashutil import hash_to_bytes, hash_to_hex | ||||
from swh.model.model import Person, Snapshot, SnapshotBranch, TargetType | from swh.model.identifiers import SWHID | ||||
from swh.model.model import ( | |||||
MetadataAuthority, | |||||
MetadataAuthorityType, | |||||
MetadataFetcher, | |||||
MetadataTargetType, | |||||
Person, | |||||
RawExtrinsicMetadata, | |||||
Snapshot, | |||||
SnapshotBranch, | |||||
TargetType, | |||||
) | |||||
from swh.storage.interface import PagedResult | |||||
from swh.loader.package import __version__ | |||||
from swh.loader.package.pypi.loader import ( | from swh.loader.package.pypi.loader import ( | ||||
PyPILoader, | PyPILoader, | ||||
pypi_api_url, | pypi_api_url, | ||||
author, | author, | ||||
extract_intrinsic_metadata, | extract_intrinsic_metadata, | ||||
artifact_to_revision_id, | artifact_to_revision_id, | ||||
) | ) | ||||
from swh.loader.package.tests.common import check_metadata_paths | from swh.loader.package.tests.common import check_metadata_paths | ||||
from swh.loader.tests import ( | from swh.loader.tests import ( | ||||
assert_last_visit_matches, | assert_last_visit_matches, | ||||
check_snapshot, | check_snapshot, | ||||
get_stats, | get_stats, | ||||
) | ) | ||||
@pytest.fixture | |||||
def _0805nexter_api_info(datadir) -> bytes: | |||||
with open( | |||||
os.path.join(datadir, "https_pypi.org", "pypi_0805nexter_json"), "rb", | |||||
) as f: | |||||
return f.read() | |||||
def test_author_basic(): | def test_author_basic(): | ||||
data = { | data = { | ||||
"author": "i-am-groot", | "author": "i-am-groot", | ||||
"author_email": "iam@groot.org", | "author_email": "iam@groot.org", | ||||
} | } | ||||
actual_author = author(data) | actual_author = author(data) | ||||
expected_author = Person( | expected_author = Person( | ||||
▲ Show 20 Lines • Show All 268 Lines • ▼ Show 20 Lines | def test_revision_metadata_structure(swh_config, requests_mock_datadir): | ||||
for original_artifact in revision["metadata"]["original_artifact"]: | for original_artifact in revision["metadata"]["original_artifact"]: | ||||
check_metadata_paths( | check_metadata_paths( | ||||
original_artifact, | original_artifact, | ||||
paths=[("filename", str), ("length", int), ("checksums", dict),], | paths=[("filename", str), ("length", int), ("checksums", dict),], | ||||
) | ) | ||||
def test_snapshot_metadata(swh_config, requests_mock_datadir, _0805nexter_api_info): | |||||
url = "https://pypi.org/project/0805nexter" | |||||
loader = PyPILoader(url) | |||||
actual_load_status = loader.load() | |||||
assert actual_load_status["status"] == "eventful" | |||||
assert actual_load_status["snapshot_id"] is not None | |||||
snapshot_swhid = SWHID( | |||||
object_type="snapshot", object_id=hash_to_hex(actual_load_status["snapshot_id"]) | |||||
) | |||||
metadata_authority = MetadataAuthority( | |||||
type=MetadataAuthorityType.FORGE, url="https://pypi.org/", | |||||
) | |||||
expected_metadata = [ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.SNAPSHOT, | |||||
id=snapshot_swhid, | |||||
authority=metadata_authority, | |||||
fetcher=MetadataFetcher( | |||||
name="swh.loader.package.pypi.loader.PyPILoader", version=__version__, | |||||
), | |||||
discovery_date=loader.visit_date, | |||||
format="pypi-project-json", | |||||
metadata=_0805nexter_api_info, | |||||
origin=url, | |||||
) | |||||
] | |||||
assert loader.storage.raw_extrinsic_metadata_get( | |||||
type=MetadataTargetType.SNAPSHOT, | |||||
id=snapshot_swhid, | |||||
authority=metadata_authority, | |||||
) == PagedResult(next_page_token=None, results=expected_metadata,) | |||||
def test_visit_with_missing_artifact(swh_config, requests_mock_datadir_missing_one): | def test_visit_with_missing_artifact(swh_config, requests_mock_datadir_missing_one): | ||||
"""Load a pypi project with some missing artifacts ends up with 1 snapshot | """Load a pypi project with some missing artifacts ends up with 1 snapshot | ||||
""" | """ | ||||
url = "https://pypi.org/project/0805nexter" | url = "https://pypi.org/project/0805nexter" | ||||
loader = PyPILoader(url) | loader = PyPILoader(url) | ||||
actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
▲ Show 20 Lines • Show All 504 Lines • Show Last 20 Lines |