diff --git a/swh/loader/package/archive/loader.py b/swh/loader/package/archive/loader.py --- a/swh/loader/package/archive/loader.py +++ b/swh/loader/package/archive/loader.py @@ -15,13 +15,7 @@ from swh.loader.package.loader import BasePackageInfo, PackageLoader, PartialExtID from swh.loader.package.utils import release_name -from swh.model.model import ( - Person, - Revision, - RevisionType, - Sha1Git, - TimestampWithTimezone, -) +from swh.model.model import ObjectType, Person, Release, Sha1Git, TimestampWithTimezone from swh.storage.interface import StorageInterface logger = logging.getLogger(__name__) @@ -148,24 +142,26 @@ ) -> Optional[PartialExtID]: return p_info.extid(manifest_format=self.extid_manifest_format) - def build_revision( - self, p_info: ArchivePackageInfo, uncompressed_path: str, directory: Sha1Git - ) -> Optional[Revision]: + def build_release( + self, + version: str, + p_info: ArchivePackageInfo, + uncompressed_path: str, + directory: Sha1Git, + ) -> Optional[Release]: time = p_info.time # assume it's a timestamp if isinstance(time, str): # otherwise, assume it's a parsable date parsed_time = iso8601.parse_date(time) else: parsed_time = time normalized_time = TimestampWithTimezone.from_datetime(parsed_time) - return Revision( - type=RevisionType.TAR, + return Release( + name=version.encode(), message=REVISION_MESSAGE, date=normalized_time, author=SWH_PERSON, - committer=SWH_PERSON, - committer_date=normalized_time, - parents=(), - directory=directory, + target=directory, + target_type=ObjectType.DIRECTORY, synthetic=True, ) diff --git a/swh/loader/package/cran/loader.py b/swh/loader/package/cran/loader.py --- a/swh/loader/package/cran/loader.py +++ b/swh/loader/package/cran/loader.py @@ -18,13 +18,7 @@ from swh.loader.package.loader import BasePackageInfo, PackageLoader from swh.loader.package.utils import release_name -from swh.model.model import ( - Person, - Revision, - RevisionType, - Sha1Git, - TimestampWithTimezone, -) +from swh.model.model import ObjectType, Person, Release, Sha1Git, TimestampWithTimezone from swh.storage.interface import StorageInterface logger = logging.getLogger(__name__) @@ -88,23 +82,25 @@ if version == p_info.version: yield release_name(version), p_info - def build_revision( - self, p_info: CRANPackageInfo, uncompressed_path: str, directory: Sha1Git - ) -> Optional[Revision]: + def build_release( + self, + version: str, + p_info: CRANPackageInfo, + uncompressed_path: str, + directory: Sha1Git, + ) -> Optional[Release]: # a_metadata is empty metadata = extract_intrinsic_metadata(uncompressed_path) date = parse_date(metadata.get("Date")) author = Person.from_fullname(metadata.get("Maintainer", "").encode()) version = metadata.get("Version", p_info.version) - return Revision( - message=version.encode("utf-8"), - type=RevisionType.TAR, + return Release( + name=version.encode(), + message=version.encode(), date=date, author=author, - committer=author, - committer_date=date, - parents=(), - directory=directory, + target_type=ObjectType.DIRECTORY, + target=directory, synthetic=True, ) diff --git a/swh/loader/package/debian/loader.py b/swh/loader/package/debian/loader.py --- a/swh/loader/package/debian/loader.py +++ b/swh/loader/package/debian/loader.py @@ -18,13 +18,7 @@ from swh.loader.package.loader import BasePackageInfo, PackageLoader, PartialExtID from swh.loader.package.utils import download, release_name from swh.model.hashutil import hash_to_bytes -from swh.model.model import ( - Person, - Revision, - RevisionType, - Sha1Git, - TimestampWithTimezone, -) +from swh.model.model import ObjectType, Person, Release, Sha1Git, TimestampWithTimezone from swh.storage.interface import StorageInterface logger = logging.getLogger(__name__) @@ -212,9 +206,13 @@ logger.debug("dl_artifacts: %s", dl_artifacts) return extract_package(dl_artifacts, dest=dest) - def build_revision( - self, p_info: DebianPackageInfo, uncompressed_path: str, directory: Sha1Git - ) -> Optional[Revision]: + def build_release( + self, + version: str, + p_info: DebianPackageInfo, + uncompressed_path: str, + directory: Sha1Git, + ) -> Optional[Release]: dsc_url, dsc_name = dsc_information(p_info) if not dsc_name: raise ValueError("dsc name for url %s should not be None" % dsc_url) @@ -234,16 +232,14 @@ author = prepare_person(intrinsic_metadata.changelog.person) date = TimestampWithTimezone.from_iso8601(intrinsic_metadata.changelog.date) - # inspired from swh.loader.debian.converters.package_metadata_to_revision # noqa - return Revision( - type=RevisionType.DSC, + # inspired from swh.loader.debian.converters.package_metadata_to_revision + return Release( + name=version.encode(), message=msg.encode("utf-8"), author=author, date=date, - committer=author, - committer_date=date, - parents=(), - directory=directory, + target=directory, + target_type=ObjectType.DIRECTORY, synthetic=True, ) diff --git a/swh/loader/package/deposit/loader.py b/swh/loader/package/deposit/loader.py --- a/swh/loader/package/deposit/loader.py +++ b/swh/loader/package/deposit/loader.py @@ -25,9 +25,9 @@ MetadataAuthority, MetadataAuthorityType, MetadataFetcher, + ObjectType, Person, - Revision, - RevisionType, + Release, Sha1Git, TimestampWithTimezone, ) @@ -188,21 +188,24 @@ """ return [self.client.archive_get(self.deposit_id, tmpdir, p_info.filename)] - def build_revision( - self, p_info: DepositPackageInfo, uncompressed_path: str, directory: Sha1Git - ) -> Optional[Revision]: + def build_release( + self, + version: str, + p_info: DepositPackageInfo, + uncompressed_path: str, + directory: Sha1Git, + ) -> Optional[Release]: message = ( f"{p_info.client}: Deposit {p_info.id} in collection {p_info.collection}" ).encode("utf-8") - return Revision( - type=RevisionType.TAR, + return Release( + name=version.encode(), message=message, author=p_info.author, date=TimestampWithTimezone.from_dict(p_info.author_date), - committer=p_info.committer, - committer_date=TimestampWithTimezone.from_dict(p_info.commit_date), - directory=directory, + target=directory, + target_type=ObjectType.DIRECTORY, synthetic=True, ) diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py --- a/swh/loader/package/loader.py +++ b/swh/loader/package/loader.py @@ -26,7 +26,6 @@ Tuple, TypeVar, ) -import warnings import attr from requests.exceptions import ContentDecodingError @@ -191,20 +190,6 @@ """ yield from {} - def build_revision( - self, p_info: TPackageInfo, uncompressed_path: str, directory: Sha1Git - ) -> Optional[Revision]: - """Build the revision from the archive metadata (extrinsic - artifact metadata) and the intrinsic metadata. - - This method is deprecated, :meth:`build_release` should be implemented instead. - - Args: - p_info: Package information - uncompressed_path: Artifact uncompressed path on disk - """ - raise NotImplementedError("build_revision") - def build_release( self, version: str, @@ -219,17 +204,7 @@ p_info: Package information uncompressed_path: Artifact uncompressed path on disk """ - warnings.warn( - f"{self.get_loader_name()} is missing a build_release() method. " - f"Falling back to `build_revision` + automatic conversion to release.", - DeprecationWarning, - ) - - rev = self.build_revision(p_info, uncompressed_path, directory) - if rev is None: - return None - else: - return rev2rel(rev, version) + raise NotImplementedError("build_release") def get_default_version(self) -> str: """Retrieve the latest release version if any. @@ -799,6 +774,7 @@ release = self.build_release( version, p_info, uncompressed_path, directory=directory.hash ) + print(release) if not release: # Some artifacts are missing intrinsic metadata # skipping those diff --git a/swh/loader/package/nixguix/loader.py b/swh/loader/package/nixguix/loader.py --- a/swh/loader/package/nixguix/loader.py +++ b/swh/loader/package/nixguix/loader.py @@ -22,8 +22,8 @@ from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, - Revision, - RevisionType, + ObjectType, + Release, Sha1Git, ) from swh.storage.interface import StorageInterface @@ -150,18 +150,20 @@ } } - def build_revision( - self, p_info: NixGuixPackageInfo, uncompressed_path: str, directory: Sha1Git - ) -> Optional[Revision]: - return Revision( - type=RevisionType.TAR, + def build_release( + self, + version: str, + p_info: NixGuixPackageInfo, + uncompressed_path: str, + directory: Sha1Git, + ) -> Optional[Release]: + return Release( + name=version.encode(), message=b"", author=EMPTY_AUTHOR, date=None, - committer=EMPTY_AUTHOR, - committer_date=None, - parents=(), - directory=directory, + target=directory, + target_type=ObjectType.DIRECTORY, synthetic=True, ) diff --git a/swh/loader/package/npm/loader.py b/swh/loader/package/npm/loader.py --- a/swh/loader/package/npm/loader.py +++ b/swh/loader/package/npm/loader.py @@ -24,9 +24,9 @@ from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, + ObjectType, Person, - Revision, - RevisionType, + Release, Sha1Git, TimestampWithTimezone, ) @@ -141,9 +141,13 @@ ) yield release_name(version), p_info - def build_revision( - self, p_info: NpmPackageInfo, uncompressed_path: str, directory: Sha1Git - ) -> Optional[Revision]: + def build_release( + self, + version: str, + p_info: NpmPackageInfo, + uncompressed_path: str, + directory: Sha1Git, + ) -> Optional[Release]: i_metadata = extract_intrinsic_metadata(uncompressed_path) if not i_metadata: return None @@ -163,15 +167,13 @@ # FIXME: this is to remain bug-compatible with earlier versions: date = attr.evolve(date, timestamp=attr.evolve(date.timestamp, microseconds=0)) - r = Revision( - type=RevisionType.TAR, + r = Release( + name=version.encode(), message=message, author=author, date=date, - committer=author, - committer_date=date, - parents=(), - directory=directory, + target=directory, + target_type=ObjectType.DIRECTORY, synthetic=True, ) return r diff --git a/swh/loader/package/opam/loader.py b/swh/loader/package/opam/loader.py --- a/swh/loader/package/opam/loader.py +++ b/swh/loader/package/opam/loader.py @@ -19,9 +19,9 @@ from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, + ObjectType, Person, - Revision, - RevisionType, + Release, Sha1Git, ) from swh.storage.interface import StorageInterface @@ -241,18 +241,20 @@ ], ) - def build_revision( - self, p_info: OpamPackageInfo, uncompressed_path: str, directory: Sha1Git - ) -> Optional[Revision]: - - return Revision( - type=RevisionType.TAR, + def build_release( + self, + version: str, + p_info: OpamPackageInfo, + uncompressed_path: str, + directory: Sha1Git, + ) -> Optional[Release]: + + return Release( + name=version.encode(), author=p_info.author, - committer=p_info.committer, message=str.encode(p_info.version), date=None, - committer_date=None, - parents=(), - directory=directory, + target=directory, + target_type=ObjectType.DIRECTORY, synthetic=True, ) diff --git a/swh/loader/package/pypi/loader.py b/swh/loader/package/pypi/loader.py --- a/swh/loader/package/pypi/loader.py +++ b/swh/loader/package/pypi/loader.py @@ -23,9 +23,9 @@ from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, + ObjectType, Person, - Revision, - RevisionType, + Release, Sha1Git, TimestampWithTimezone, ) @@ -125,31 +125,33 @@ for version, p_info in res: yield release_name(version, p_info.filename), p_info - def build_revision( - self, p_info: PyPIPackageInfo, uncompressed_path: str, directory: Sha1Git - ) -> Optional[Revision]: + def build_release( + self, + version: str, + p_info: PyPIPackageInfo, + uncompressed_path: str, + directory: Sha1Git, + ) -> Optional[Release]: i_metadata = extract_intrinsic_metadata(uncompressed_path) if not i_metadata: return None # from intrinsic metadata - version = i_metadata.get("version", "") - _author = author(i_metadata) + version_ = i_metadata.get("version", "") + author_ = author(i_metadata) # from extrinsic metadata message = p_info.comment_text or "" - message = "%s: %s" % (version, message) if message else version + message = "%s: %s" % (version_, message) if message else version_ date = TimestampWithTimezone.from_iso8601(p_info.upload_time) - return Revision( - type=RevisionType.TAR, - message=message.encode("utf-8"), - author=_author, + return Release( + name=version.encode(), + message=message.encode(), + author=author_, date=date, - committer=_author, - committer_date=date, - parents=(), - directory=directory, + target=directory, + target_type=ObjectType.DIRECTORY, synthetic=True, ) diff --git a/swh/loader/package/tests/test_loader_metadata.py b/swh/loader/package/tests/test_loader_metadata.py --- a/swh/loader/package/tests/test_loader_metadata.py +++ b/swh/loader/package/tests/test_loader_metadata.py @@ -19,14 +19,14 @@ MetadataAuthority, MetadataAuthorityType, MetadataFetcher, + ObjectType, Origin, Person, RawExtrinsicMetadata, - Revision, - RevisionType, + Release, Sha1Git, ) -from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID, ObjectType +from swh.model.swhids import CoreSWHID, ExtendedSWHID EMPTY_SNAPSHOT_ID = "1a8893e6a86f444e8be8e7bda6cb34fb1735a00e" FULL_SNAPSHOT_ID = "4ac5730a9393f5099b63a35a17b6c33d36d70c3a" @@ -39,11 +39,9 @@ REVISION_ID = hash_to_bytes("8ff44f081d43176474b267de5451f2c2e88089d0") RELEASE_ID = hash_to_bytes("9477a708196b44e59efb4e47b7d979a4146bd428") -RELEASE_SWHID = CoreSWHID(object_type=ObjectType.RELEASE, object_id=RELEASE_ID) +RELEASE_SWHID = CoreSWHID.from_string(f"swh:1:rel:{RELEASE_ID.hex()}") DIRECTORY_ID = hash_to_bytes("aa" * 20) -DIRECTORY_SWHID = ExtendedSWHID( - object_type=ExtendedObjectType.DIRECTORY, object_id=DIRECTORY_ID -) +DIRECTORY_SWHID = ExtendedSWHID.from_string(f"swh:1:dir:{DIRECTORY_ID.hex()}") FETCHER = MetadataFetcher( @@ -101,18 +99,20 @@ def download_package(self, p_info: BasePackageInfo, tmpdir: str): return [("path", {"artifact_key": "value", "length": 0})] - def build_revision( - self, p_info: BasePackageInfo, uncompressed_path: str, directory: Sha1Git + def build_release( + self, + version: str, + p_info: BasePackageInfo, + uncompressed_path: str, + directory: Sha1Git, ): - return Revision( - id=REVISION_ID, + return Release( + name=version.encode(), message=b"", author=Person.from_fullname(b""), - committer=Person.from_fullname(b""), date=None, - committer_date=None, - type=RevisionType.TAR, - directory=DIRECTORY_ID, + target=DIRECTORY_ID, + target_type=ObjectType.DIRECTORY, synthetic=False, )