diff --git a/swh/loader/package/maven/loader.py b/swh/loader/package/maven/loader.py index ef5e456..7c85288 100644 --- a/swh/loader/package/maven/loader.py +++ b/swh/loader/package/maven/loader.py @@ -1,212 +1,215 @@ # Copyright (C) 2021-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import annotations from datetime import datetime, timezone import json import logging from os import path import string from typing import Any, Iterator, List, Optional, Sequence, Tuple import attr import iso8601 import requests from typing_extensions import TypedDict +from swh.loader.exception import NotFound from swh.loader.package.loader import ( BasePackageInfo, PackageLoader, RawExtrinsicMetadataCore, ) from swh.loader.package.utils import EMPTY_AUTHOR, get_url_body, release_name from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, ObjectType, RawExtrinsicMetadata, Release, Sha1Git, TimestampWithTimezone, ) from swh.storage.interface import StorageInterface logger = logging.getLogger(__name__) class ArtifactDict(TypedDict): """Data about a Maven artifact, passed by the Maven Lister.""" time: str """the time of the last update of jar file on the server as an iso8601 date string """ url: str """the artifact url to retrieve filename""" filename: Optional[str] """optionally, the file's name""" gid: str """artifact's groupId""" aid: str """artifact's artifactId""" version: str """artifact's version""" base_url: str """root URL of the Maven instance""" @attr.s class MavenPackageInfo(BasePackageInfo): time = attr.ib(type=datetime) """Timestamp of the last update of jar file on the server.""" gid = attr.ib(type=str) """Group ID of the maven artifact""" aid = attr.ib(type=str) """Artifact ID of the maven artifact""" version = attr.ib(type=str) """Version of the maven artifact""" base_url = attr.ib(type=str) """Root URL of the Maven instance""" # default format for maven artifacts MANIFEST_FORMAT = string.Template("$gid $aid $version $url $time") EXTID_TYPE = "maven-jar" EXTID_VERSION = 0 @classmethod def from_metadata(cls, a_metadata: ArtifactDict) -> MavenPackageInfo: time = iso8601.parse_date(a_metadata["time"]).astimezone(tz=timezone.utc) url = a_metadata["url"] checksums = {} - try: - checksums["sha1"] = get_url_body(url + ".sha1").decode() - except requests.HTTPError: - pass + for algo in ("sha1", "md5"): + try: + checksums[algo] = get_url_body(url + f".{algo}").decode() + break + except (requests.HTTPError, NotFound): + pass return cls( url=url, filename=a_metadata.get("filename") or path.split(url)[-1], time=time, gid=a_metadata["gid"], aid=a_metadata["aid"], version=a_metadata["version"], base_url=a_metadata["base_url"], directory_extrinsic_metadata=[ RawExtrinsicMetadataCore( format="maven-json", metadata=json.dumps(a_metadata).encode(), ), ], checksums=checksums, ) class MavenLoader(PackageLoader[MavenPackageInfo]): """Load source code jar origin's artifact files into swh archive""" visit_type = "maven" def __init__( self, storage: StorageInterface, url: str, artifacts: Sequence[ArtifactDict], **kwargs: Any, ): """Loader constructor. For now, this is the lister's task output. There is one, and only one, artefact (jar or zip) per version, as guaranteed by the Maven coordinates system. Args: url: Origin url artifacts: List of single artifact information """ super().__init__(storage=storage, url=url, **kwargs) self.artifacts = artifacts # assume order is enforced in the lister self.version_artifact = { jar["version"]: jar for jar in artifacts if jar["version"] } if artifacts: base_urls = {jar["base_url"] for jar in artifacts} try: (self.base_url,) = base_urls except ValueError: raise ValueError( "Artifacts originate from more than one Maven instance: " + ", ".join(base_urls) ) from None else: # There is no artifact, so self.metadata_authority won't be called, # so self.base_url won't be accessed. pass def get_versions(self) -> Sequence[str]: return list(self.version_artifact) def get_default_version(self) -> str: # Default version is the one of the most recent artifact return max(self.artifacts, key=lambda a: a["time"])["version"] def get_metadata_authority(self): return MetadataAuthority(type=MetadataAuthorityType.FORGE, url=self.base_url) def build_extrinsic_directory_metadata( self, p_info: MavenPackageInfo, release_id: Sha1Git, directory_id: Sha1Git, ) -> List[RawExtrinsicMetadata]: # Rebuild POM URL. pom_url = path.dirname(p_info.url) pom_url = f"{pom_url}/{p_info.aid}-{p_info.version}.pom" r = requests.get(pom_url, allow_redirects=True) if r.status_code == 200: metadata_pom = r.content else: metadata_pom = b"" p_info.directory_extrinsic_metadata.append( RawExtrinsicMetadataCore( format="maven-pom", metadata=metadata_pom, ) ) return super().build_extrinsic_directory_metadata( p_info=p_info, release_id=release_id, directory_id=directory_id, ) def get_package_info(self, version: str) -> Iterator[Tuple[str, MavenPackageInfo]]: a_metadata = self.version_artifact[version] rel_name = release_name(a_metadata["version"]) yield rel_name, MavenPackageInfo.from_metadata(a_metadata) def build_release( self, p_info: MavenPackageInfo, uncompressed_path: str, directory: Sha1Git ) -> Optional[Release]: msg = f"Synthetic release for archive at {p_info.url}\n".encode("utf-8") normalized_time = TimestampWithTimezone.from_datetime(p_info.time) return Release( name=p_info.version.encode(), message=msg, date=normalized_time, author=EMPTY_AUTHOR, target=directory, target_type=ObjectType.DIRECTORY, synthetic=True, ) diff --git a/swh/loader/package/maven/tests/data/https_repo1.maven.org/maven2_al_aldi_sprova4j_0.1.0_sprova4j-0.1.0-sources.jar.md5 b/swh/loader/package/maven/tests/data/https_repo1.maven.org/maven2_al_aldi_sprova4j_0.1.0_sprova4j-0.1.0-sources.jar.md5 new file mode 100644 index 0000000..0cd6563 --- /dev/null +++ b/swh/loader/package/maven/tests/data/https_repo1.maven.org/maven2_al_aldi_sprova4j_0.1.0_sprova4j-0.1.0-sources.jar.md5 @@ -0,0 +1 @@ +c380281ccea02c95519a77dd96112d59 \ No newline at end of file diff --git a/swh/loader/package/maven/tests/data/https_repo1.maven.org/maven2_al_aldi_sprova4j_0.1.0_sprova4j-0.1.0-sources.jar.sha1 b/swh/loader/package/maven/tests/data/https_repo1.maven.org/maven2_al_aldi_sprova4j_0.1.0_sprova4j-0.1.0-sources.jar.sha1 deleted file mode 100644 index 9b4204d..0000000 --- a/swh/loader/package/maven/tests/data/https_repo1.maven.org/maven2_al_aldi_sprova4j_0.1.0_sprova4j-0.1.0-sources.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6976e186000753610a63713677f42f0228f04e64 \ No newline at end of file diff --git a/swh/loader/package/maven/tests/test_maven.py b/swh/loader/package/maven/tests/test_maven.py index acf88fb..e7b8c2e 100644 --- a/swh/loader/package/maven/tests/test_maven.py +++ b/swh/loader/package/maven/tests/test_maven.py @@ -1,456 +1,462 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import datetime import hashlib from itertools import chain import json import os import pytest import requests from swh.core.tarball import uncompress from swh.loader.package import __version__ from swh.loader.package.maven.loader import MavenLoader, MavenPackageInfo from swh.loader.package.utils import EMPTY_AUTHOR from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats from swh.model.from_disk import Directory, iter_directory from swh.model.hashutil import hash_to_bytes from swh.model.model import ( RawExtrinsicMetadata, Release, Snapshot, SnapshotBranch, TargetType, TimestampWithTimezone, ) from swh.model.model import MetadataAuthority, MetadataAuthorityType, MetadataFetcher from swh.model.model import ObjectType as ModelObjectType from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID, ObjectType from swh.storage.algos.snapshot import snapshot_get_all_branches REPO_BASE_URL = "https://repo1.maven.org/maven2/" MVN_ORIGIN_URL = f"{REPO_BASE_URL}al/aldi/sprova4j" MVN_ARTIFACTS = [ { "time": "2021-07-12 19:06:59.335000", "gid": "al.aldi", "aid": "sprova4j", "filename": "sprova4j-0.1.0-sources.jar", "version": "0.1.0", "base_url": REPO_BASE_URL, "url": f"{REPO_BASE_URL}al/aldi/sprova4j/0.1.0/sprova4j-0.1.0-sources.jar", }, { "time": "2021-07-12 19:37:05.534000", "gid": "al.aldi", "aid": "sprova4j", "filename": "sprova4j-0.1.1-sources.jar", "version": "0.1.1", "base_url": REPO_BASE_URL, "url": f"{REPO_BASE_URL}al/aldi/sprova4j/0.1.1/sprova4j-0.1.1-sources.jar", }, ] MVN_ARTIFACTS_POM = [ f"{REPO_BASE_URL}al/aldi/sprova4j/0.1.0/sprova4j-0.1.0.pom", f"{REPO_BASE_URL}al/aldi/sprova4j/0.1.1/sprova4j-0.1.1.pom", ] REL_MSGS = ( b"Synthetic release for archive at https://repo1.maven.org/maven2/al/aldi/" b"sprova4j/0.1.0/sprova4j-0.1.0-sources.jar\n", b"Synthetic release for archive at https://repo1.maven.org/maven2/al/aldi/" b"sprova4j/0.1.1/sprova4j-0.1.1-sources.jar\n", ) REL_DATES = ( TimestampWithTimezone.from_datetime( datetime.datetime(2021, 7, 12, 19, 6, 59, 335000, tzinfo=datetime.timezone.utc) ), TimestampWithTimezone.from_datetime( datetime.datetime(2021, 7, 12, 19, 37, 5, 534000, tzinfo=datetime.timezone.utc) ), ) @pytest.fixture(autouse=True) def network_requests_mock(requests_mock_datadir): pass @pytest.fixture def jar_dirs(tmp_path): jar_1_path = os.path.join(tmp_path, os.path.basename(MVN_ARTIFACTS[0]["url"])) jar_2_path = os.path.join(tmp_path, os.path.basename(MVN_ARTIFACTS[1]["url"])) with open(jar_1_path, "wb") as jar_1, open(jar_2_path, "wb") as jar_2: jar_1.write(requests.get(MVN_ARTIFACTS[0]["url"]).content) jar_2.write(requests.get(MVN_ARTIFACTS[1]["url"]).content) jar_1_extract_path = os.path.join(tmp_path, "jar_1") jar_2_extract_path = os.path.join(tmp_path, "jar_2") uncompress(jar_1_path, jar_1_extract_path) uncompress(jar_2_path, jar_2_extract_path) jar_1_dir = Directory.from_disk(path=jar_1_extract_path.encode()) jar_2_dir = Directory.from_disk(path=jar_2_extract_path.encode()) return [jar_1_dir, jar_2_dir] @pytest.fixture def expected_contents_and_directories(jar_dirs): jar_1_cnts, _, jar_1_dirs = iter_directory(jar_dirs[0]) jar_2_cnts, _, jar_2_dirs = iter_directory(jar_dirs[1]) contents = {cnt.sha1 for cnt in chain(jar_1_cnts, jar_2_cnts)} directories = {dir.id for dir in chain(jar_1_dirs, jar_2_dirs)} return contents, directories @pytest.fixture def expected_releases(jar_dirs): return [ Release( name=b"0.1.0", message=REL_MSGS[0], author=EMPTY_AUTHOR, date=REL_DATES[0], target_type=ModelObjectType.DIRECTORY, target=jar_dirs[0].hash, synthetic=True, metadata=None, ), Release( name=b"0.1.1", message=REL_MSGS[1], author=EMPTY_AUTHOR, date=REL_DATES[1], target_type=ModelObjectType.DIRECTORY, target=jar_dirs[1].hash, synthetic=True, metadata=None, ), ] @pytest.fixture def expected_snapshot(expected_releases): return Snapshot( branches={ b"HEAD": SnapshotBranch( target_type=TargetType.ALIAS, target=b"releases/0.1.1", ), b"releases/0.1.0": SnapshotBranch( target_type=TargetType.RELEASE, target=expected_releases[0].id, ), b"releases/0.1.1": SnapshotBranch( target_type=TargetType.RELEASE, target=expected_releases[1].id, ), }, ) @pytest.fixture def expected_json_metadata(): return MVN_ARTIFACTS @pytest.fixture def expected_pom_metadata(): return [requests.get(pom_url).content for pom_url in MVN_ARTIFACTS_POM] def test_maven_loader_visit_with_no_artifact_found(swh_storage): origin_url = "https://ftp.g.o/unknown" unknown_artifact_url = "https://ftp.g.o/unknown/8sync-0.1.0.tar.gz" loader = MavenLoader( swh_storage, origin_url, artifacts=[ { "time": "2021-07-18 08:05:05.187000", "url": unknown_artifact_url, # unknown artifact "filename": "8sync-0.1.0.tar.gz", "gid": "al/aldi", "aid": "sprova4j", "version": "0.1.0", "base_url": "https://repo1.maven.org/maven2/", } ], ) actual_load_status = loader.load() assert actual_load_status["status"] == "uneventful" assert actual_load_status["snapshot_id"] is not None expected_snapshot_id = "1a8893e6a86f444e8be8e7bda6cb34fb1735a00e" assert actual_load_status["snapshot_id"] == expected_snapshot_id stats = get_stats(swh_storage) assert_last_visit_matches(swh_storage, origin_url, status="partial", type="maven") assert { "content": 0, "directory": 0, "origin": 1, "origin_visit": 1, "release": 0, "revision": 0, "skipped_content": 0, "snapshot": 1, } == stats def test_maven_loader_jar_visit_inconsistent_base_url(swh_storage): """With no prior visit, loading a jar ends up with 1 snapshot""" with pytest.raises(ValueError, match="more than one Maven instance"): MavenLoader( swh_storage, MVN_ORIGIN_URL, artifacts=[ MVN_ARTIFACTS[0], {**MVN_ARTIFACTS[1], "base_url": "http://maven.example/"}, ], ) def test_maven_loader_first_visit( swh_storage, expected_contents_and_directories, expected_snapshot, expected_releases, ): """With no prior visit, loading a jar ends up with 1 snapshot""" loader = MavenLoader(swh_storage, MVN_ORIGIN_URL, artifacts=MVN_ARTIFACTS) actual_load_status = loader.load() assert actual_load_status["status"] == "eventful" actual_snapshot = snapshot_get_all_branches( swh_storage, hash_to_bytes(actual_load_status["snapshot_id"]) ) assert actual_load_status["snapshot_id"] == expected_snapshot.id.hex() check_snapshot(expected_snapshot, swh_storage) stats = get_stats(swh_storage) assert_last_visit_matches(swh_storage, MVN_ORIGIN_URL, status="full", type="maven") expected_contents, expected_directories = expected_contents_and_directories assert list(swh_storage.content_missing_per_sha1(expected_contents)) == [] assert list(swh_storage.directory_missing(expected_directories)) == [] rel_id = actual_snapshot.branches[b"releases/0.1.0"].target rel2_id = actual_snapshot.branches[b"releases/0.1.1"].target releases = swh_storage.release_get([rel_id, rel2_id]) assert releases == expected_releases assert { "content": len(expected_contents), "directory": len(expected_directories), "origin": 1, "origin_visit": 1, "release": 2, "revision": 0, "skipped_content": 0, "snapshot": 1, } == stats + for version in loader.get_versions(): + _, package_info = next(loader.get_package_info(version)) + assert package_info.checksums + def test_maven_loader_2_visits_without_change( swh_storage, requests_mock, expected_snapshot ): """With no prior visit, load a maven project ends up with 1 snapshot""" # reset requests history as some are sent by fixtures requests_mock.reset_mock() loader = MavenLoader(swh_storage, MVN_ORIGIN_URL, artifacts=MVN_ARTIFACTS) actual_load_status = loader.load() assert actual_load_status["status"] == "eventful" assert actual_load_status["snapshot_id"] == expected_snapshot.id.hex() assert_last_visit_matches(swh_storage, MVN_ORIGIN_URL, status="full", type="maven") actual_load_status2 = loader.load() assert actual_load_status2["status"] == "uneventful" assert actual_load_status2["snapshot_id"] is not None assert actual_load_status["snapshot_id"] == actual_load_status2["snapshot_id"] assert_last_visit_matches(swh_storage, MVN_ORIGIN_URL, status="full", type="maven") # Make sure we have only one entry in history for the pom fetch, one for # the actual download of jar, and that they're correct. urls_history = [str(req.url) for req in list(requests_mock.request_history)] assert urls_history == [ MVN_ARTIFACTS[0]["url"] + ".sha1", + MVN_ARTIFACTS[0]["url"] + ".md5", MVN_ARTIFACTS[1]["url"] + ".sha1", MVN_ARTIFACTS[0]["url"], MVN_ARTIFACTS_POM[0], MVN_ARTIFACTS[1]["url"], MVN_ARTIFACTS_POM[1], MVN_ARTIFACTS[0]["url"] + ".sha1", + MVN_ARTIFACTS[0]["url"] + ".md5", MVN_ARTIFACTS[1]["url"] + ".sha1", ] def test_maven_loader_extrinsic_metadata( swh_storage, expected_releases, expected_json_metadata, expected_pom_metadata, ): """With no prior visit, loading a jar ends up with 1 snapshot. Extrinsic metadata is the pom file associated to the source jar. """ loader = MavenLoader(swh_storage, MVN_ORIGIN_URL, artifacts=MVN_ARTIFACTS) actual_load_status = loader.load() assert actual_load_status["status"] == "eventful" for i, expected_release in enumerate(expected_releases): expected_release_id = expected_release.id release = swh_storage.release_get([expected_release_id])[0] assert release is not None release_swhid = CoreSWHID( object_type=ObjectType.RELEASE, object_id=expected_release_id ) directory_swhid = ExtendedSWHID( object_type=ExtendedObjectType.DIRECTORY, object_id=release.target ) metadata_authority = MetadataAuthority( type=MetadataAuthorityType.FORGE, url=REPO_BASE_URL, ) expected_metadata = [ RawExtrinsicMetadata( target=directory_swhid, authority=metadata_authority, fetcher=MetadataFetcher( name="swh.loader.package.maven.loader.MavenLoader", version=__version__, ), discovery_date=loader.visit_date, format="maven-pom", metadata=expected_pom_metadata[i], origin=MVN_ORIGIN_URL, release=release_swhid, ), RawExtrinsicMetadata( target=directory_swhid, authority=metadata_authority, fetcher=MetadataFetcher( name="swh.loader.package.maven.loader.MavenLoader", version=__version__, ), discovery_date=loader.visit_date, format="maven-json", metadata=json.dumps(expected_json_metadata[i]).encode(), origin=MVN_ORIGIN_URL, release=release_swhid, ), ] res = swh_storage.raw_extrinsic_metadata_get( directory_swhid, metadata_authority ) assert res.next_page_token is None assert set(res.results) == set(expected_metadata) def test_maven_loader_extrinsic_metadata_no_pom( swh_storage, requests_mock, expected_releases, expected_json_metadata, ): """With no prior visit, loading a jar ends up with 1 snapshot. Extrinsic metadata is None if the pom file cannot be retrieved. """ requests_mock.get(MVN_ARTIFACTS_POM[0], status_code="404") loader = MavenLoader(swh_storage, MVN_ORIGIN_URL, artifacts=MVN_ARTIFACTS) actual_load_status = loader.load() assert actual_load_status["status"] == "eventful" expected_release_id = expected_releases[0].id release = swh_storage.release_get([expected_release_id])[0] assert release is not None release_swhid = CoreSWHID( object_type=ObjectType.RELEASE, object_id=expected_release_id ) directory_swhid = ExtendedSWHID( object_type=ExtendedObjectType.DIRECTORY, object_id=release.target ) metadata_authority = MetadataAuthority( type=MetadataAuthorityType.FORGE, url=REPO_BASE_URL, ) expected_metadata = [ RawExtrinsicMetadata( target=directory_swhid, authority=metadata_authority, fetcher=MetadataFetcher( name="swh.loader.package.maven.loader.MavenLoader", version=__version__, ), discovery_date=loader.visit_date, format="maven-pom", metadata=b"", origin=MVN_ORIGIN_URL, release=release_swhid, ), RawExtrinsicMetadata( target=directory_swhid, authority=metadata_authority, fetcher=MetadataFetcher( name="swh.loader.package.maven.loader.MavenLoader", version=__version__, ), discovery_date=loader.visit_date, format="maven-json", metadata=json.dumps(expected_json_metadata[0]).encode(), origin=MVN_ORIGIN_URL, release=release_swhid, ), ] res = swh_storage.raw_extrinsic_metadata_get(directory_swhid, metadata_authority) assert res.next_page_token is None assert set(res.results) == set(expected_metadata) def test_maven_loader_jar_extid(): """Compute primary key should return the right identity""" metadata = MVN_ARTIFACTS[0] p_info = MavenPackageInfo(**metadata) expected_manifest = "{gid} {aid} {version} {url} {time}".format(**metadata).encode() actual_id = p_info.extid() assert actual_id == ( "maven-jar", 0, hashlib.sha256(expected_manifest).digest(), )