Differential D3677 Diff 12947 swh/loader/package/pypi/tests/test_pypi.py

Changeset View

Standalone View

swh/loader/package/pypi/tests/test_pypi.py

# Copyright (C) 2019-2020 The Software Heritage developers		# Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import os		import os

from os import path		from os import path

import pytest		import pytest

from unittest.mock import patch		from unittest.mock import patch

from swh.core.tarball import uncompress		from swh.core.tarball import uncompress
from swh.core.pytest_plugin import requests_mock_datadir_factory		from swh.core.pytest_plugin import requests_mock_datadir_factory
from swh.model.hashutil import hash_to_bytes		from swh.model.hashutil import hash_to_bytes, hash_to_hex
from swh.model.model import Person, Snapshot, SnapshotBranch, TargetType		from swh.model.identifiers import SWHID
		from swh.model.model import (
		MetadataAuthority,
		MetadataAuthorityType,
		MetadataFetcher,
		MetadataTargetType,
		Person,
		RawExtrinsicMetadata,
		Snapshot,
		SnapshotBranch,
		TargetType,
		)
		from swh.storage.interface import PagedResult

		from swh.loader.package import __version__
from swh.loader.package.pypi.loader import (		from swh.loader.package.pypi.loader import (
PyPILoader,		PyPILoader,
pypi_api_url,		pypi_api_url,
author,		author,
extract_intrinsic_metadata,		extract_intrinsic_metadata,
artifact_to_revision_id,		artifact_to_revision_id,
)		)
from swh.loader.package.tests.common import check_metadata_paths		from swh.loader.package.tests.common import check_metadata_paths
from swh.loader.tests import (		from swh.loader.tests import (
assert_last_visit_matches,		assert_last_visit_matches,
check_snapshot,		check_snapshot,
get_stats,		get_stats,
)		)


		@pytest.fixture
		def _0805nexter_api_info(datadir) -> bytes:
		with open(
		os.path.join(datadir, "https_pypi.org", "pypi_0805nexter_json"), "rb",
		) as f:
		return f.read()


def test_author_basic():		def test_author_basic():
data = {		data = {
"author": "i-am-groot",		"author": "i-am-groot",
"author_email": "iam@groot.org",		"author_email": "iam@groot.org",
}		}
actual_author = author(data)		actual_author = author(data)

expected_author = Person(		expected_author = Person(
▲ Show 20 Lines • Show All 268 Lines • ▼ Show 20 Lines	def test_revision_metadata_structure(swh_config, requests_mock_datadir):

for original_artifact in revision["metadata"]["original_artifact"]:		for original_artifact in revision["metadata"]["original_artifact"]:
check_metadata_paths(		check_metadata_paths(
original_artifact,		original_artifact,
paths=[("filename", str), ("length", int), ("checksums", dict),],		paths=[("filename", str), ("length", int), ("checksums", dict),],
)		)


		def test_snapshot_metadata(swh_config, requests_mock_datadir, _0805nexter_api_info):
		url = "https://pypi.org/project/0805nexter"
		loader = PyPILoader(url)

		actual_load_status = loader.load()
		assert actual_load_status["status"] == "eventful"
		assert actual_load_status["snapshot_id"] is not None

		snapshot_swhid = SWHID(
		object_type="snapshot", object_id=hash_to_hex(actual_load_status["snapshot_id"])
		)
		metadata_authority = MetadataAuthority(
		type=MetadataAuthorityType.FORGE, url="https://pypi.org/",
		)
		expected_metadata = [
		RawExtrinsicMetadata(
		type=MetadataTargetType.SNAPSHOT,
		id=snapshot_swhid,
		authority=metadata_authority,
		fetcher=MetadataFetcher(
		name="swh.loader.package.pypi.loader.PyPILoader", version=__version__,
		),
		discovery_date=loader.visit_date,
		format="pypi-project-json",
		metadata=_0805nexter_api_info,
		origin=url,
		)
		]
		assert loader.storage.raw_extrinsic_metadata_get(
		type=MetadataTargetType.SNAPSHOT,
		id=snapshot_swhid,
		authority=metadata_authority,
		) == PagedResult(next_page_token=None, results=expected_metadata,)


def test_visit_with_missing_artifact(swh_config, requests_mock_datadir_missing_one):		def test_visit_with_missing_artifact(swh_config, requests_mock_datadir_missing_one):
"""Load a pypi project with some missing artifacts ends up with 1 snapshot		"""Load a pypi project with some missing artifacts ends up with 1 snapshot

"""		"""
url = "https://pypi.org/project/0805nexter"		url = "https://pypi.org/project/0805nexter"
loader = PyPILoader(url)		loader = PyPILoader(url)

actual_load_status = loader.load()		actual_load_status = loader.load()
▲ Show 20 Lines • Show All 504 Lines • Show Last 20 Lines