Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/loader.py
Show First 20 Lines • Show All 443 Lines • ▼ Show 20 Lines | def load(self) -> Dict: | ||||
) | ) | ||||
self.storage.flush() | self.storage.flush() | ||||
except Exception as e: | except Exception as e: | ||||
logger.exception("Failed to build snapshot for origin %s", self.url) | logger.exception("Failed to build snapshot for origin %s", self.url) | ||||
sentry_sdk.capture_exception(e) | sentry_sdk.capture_exception(e) | ||||
status_visit = "partial" | status_visit = "partial" | ||||
status_load = "failed" | status_load = "failed" | ||||
if snapshot: | |||||
try: | |||||
metadata_objects = self.build_extrinsic_snapshot_metadata(snapshot.id) | |||||
self._load_metadata_objects(metadata_objects) | |||||
except Exception as e: | |||||
logger.exception( | |||||
"Failed to load extrinsic snapshot metadata for %s", self.url | |||||
) | |||||
sentry_sdk.capture_exception(e) | |||||
status_visit = "partial" | |||||
status_load = "failed" | |||||
try: | try: | ||||
metadata_objects = self.build_extrinsic_origin_metadata() | metadata_objects = self.build_extrinsic_origin_metadata() | ||||
self._load_metadata_objects(metadata_objects) | self._load_metadata_objects(metadata_objects) | ||||
except Exception as e: | except Exception as e: | ||||
logger.exception( | logger.exception( | ||||
"Failed to load extrinsic origin metadata for %s", self.url | "Failed to load extrinsic origin metadata for %s", self.url | ||||
) | ) | ||||
sentry_sdk.capture_exception(e) | sentry_sdk.capture_exception(e) | ||||
▲ Show 20 Lines • Show All 173 Lines • ▼ Show 20 Lines | def build_extrinsic_origin_metadata(self) -> List[RawExtrinsicMetadata]: | ||||
fetcher=fetcher, | fetcher=fetcher, | ||||
format=item.format, | format=item.format, | ||||
metadata=item.metadata, | metadata=item.metadata, | ||||
) | ) | ||||
) | ) | ||||
return metadata_objects | return metadata_objects | ||||
def get_extrinsic_snapshot_metadata(self) -> List[RawExtrinsicMetadataCore]: | |||||
"""Returns metadata items, used by build_extrinsic_snapshot_metadata.""" | |||||
return [] | |||||
def build_extrinsic_snapshot_metadata( | |||||
self, snapshot_id: Sha1Git | |||||
) -> List[RawExtrinsicMetadata]: | |||||
"""Builds a list of full RawExtrinsicMetadata objects, using | |||||
metadata returned by get_extrinsic_snapshot_metadata.""" | |||||
metadata_items = self.get_extrinsic_snapshot_metadata() | |||||
if not metadata_items: | |||||
# If this package loader doesn't write metadata, no need to require | |||||
# an implementation for get_metadata_authority. | |||||
return [] | |||||
authority = self.get_metadata_authority() | |||||
fetcher = self.get_metadata_fetcher() | |||||
metadata_objects = [] | |||||
for item in metadata_items: | |||||
metadata_objects.append( | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.SNAPSHOT, | |||||
id=SWHID(object_type="snapshot", object_id=snapshot_id), | |||||
discovery_date=item.discovery_date or self.visit_date, | |||||
authority=authority, | |||||
fetcher=fetcher, | |||||
format=item.format, | |||||
metadata=item.metadata, | |||||
origin=self.url, | |||||
) | |||||
) | |||||
return metadata_objects | |||||
def build_extrinsic_revision_metadata( | def build_extrinsic_revision_metadata( | ||||
self, p_info: TPackageInfo, revision_id: Sha1Git | self, p_info: TPackageInfo, revision_id: Sha1Git | ||||
) -> List[RawExtrinsicMetadata]: | ) -> List[RawExtrinsicMetadata]: | ||||
if not p_info.revision_extrinsic_metadata: | if not p_info.revision_extrinsic_metadata: | ||||
# If this package loader doesn't write metadata, no need to require | # If this package loader doesn't write metadata, no need to require | ||||
# an implementation for get_metadata_authority. | # an implementation for get_metadata_authority. | ||||
return [] | return [] | ||||
▲ Show 20 Lines • Show All 53 Lines • Show Last 20 Lines |