Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/loader.py
| Show First 20 Lines • Show All 443 Lines • ▼ Show 20 Lines | def load(self) -> Dict: | ||||
| ) | ) | ||||
| self.storage.flush() | self.storage.flush() | ||||
| except Exception as e: | except Exception as e: | ||||
| logger.exception("Failed to build snapshot for origin %s", self.url) | logger.exception("Failed to build snapshot for origin %s", self.url) | ||||
| sentry_sdk.capture_exception(e) | sentry_sdk.capture_exception(e) | ||||
| status_visit = "partial" | status_visit = "partial" | ||||
| status_load = "failed" | status_load = "failed" | ||||
| if snapshot: | |||||
| try: | |||||
| metadata_objects = self.build_extrinsic_snapshot_metadata(snapshot.id) | |||||
| self._load_metadata_objects(metadata_objects) | |||||
| except Exception as e: | |||||
| logger.exception( | |||||
| "Failed to load extrinsic snapshot metadata for %s", self.url | |||||
| ) | |||||
| sentry_sdk.capture_exception(e) | |||||
| status_visit = "partial" | |||||
| status_load = "failed" | |||||
| try: | try: | ||||
| metadata_objects = self.build_extrinsic_origin_metadata() | metadata_objects = self.build_extrinsic_origin_metadata() | ||||
| self._load_metadata_objects(metadata_objects) | self._load_metadata_objects(metadata_objects) | ||||
| except Exception as e: | except Exception as e: | ||||
| logger.exception( | logger.exception( | ||||
| "Failed to load extrinsic origin metadata for %s", self.url | "Failed to load extrinsic origin metadata for %s", self.url | ||||
| ) | ) | ||||
| sentry_sdk.capture_exception(e) | sentry_sdk.capture_exception(e) | ||||
| ▲ Show 20 Lines • Show All 173 Lines • ▼ Show 20 Lines | def build_extrinsic_origin_metadata(self) -> List[RawExtrinsicMetadata]: | ||||
| fetcher=fetcher, | fetcher=fetcher, | ||||
| format=item.format, | format=item.format, | ||||
| metadata=item.metadata, | metadata=item.metadata, | ||||
| ) | ) | ||||
| ) | ) | ||||
| return metadata_objects | return metadata_objects | ||||
| def get_extrinsic_snapshot_metadata(self) -> List[RawExtrinsicMetadataCore]: | |||||
| """Returns metadata items, used by build_extrinsic_snapshot_metadata.""" | |||||
| return [] | |||||
| def build_extrinsic_snapshot_metadata( | |||||
| self, snapshot_id: Sha1Git | |||||
| ) -> List[RawExtrinsicMetadata]: | |||||
| """Builds a list of full RawExtrinsicMetadata objects, using | |||||
| metadata returned by get_extrinsic_snapshot_metadata.""" | |||||
| metadata_items = self.get_extrinsic_snapshot_metadata() | |||||
| if not metadata_items: | |||||
| # If this package loader doesn't write metadata, no need to require | |||||
| # an implementation for get_metadata_authority. | |||||
| return [] | |||||
| authority = self.get_metadata_authority() | |||||
| fetcher = self.get_metadata_fetcher() | |||||
| metadata_objects = [] | |||||
| for item in metadata_items: | |||||
| metadata_objects.append( | |||||
| RawExtrinsicMetadata( | |||||
| type=MetadataTargetType.SNAPSHOT, | |||||
| id=SWHID(object_type="snapshot", object_id=snapshot_id), | |||||
| discovery_date=item.discovery_date or self.visit_date, | |||||
| authority=authority, | |||||
| fetcher=fetcher, | |||||
| format=item.format, | |||||
| metadata=item.metadata, | |||||
| origin=self.url, | |||||
| ) | |||||
| ) | |||||
| return metadata_objects | |||||
| def build_extrinsic_revision_metadata( | def build_extrinsic_revision_metadata( | ||||
| self, p_info: TPackageInfo, revision_id: Sha1Git | self, p_info: TPackageInfo, revision_id: Sha1Git | ||||
| ) -> List[RawExtrinsicMetadata]: | ) -> List[RawExtrinsicMetadata]: | ||||
| if not p_info.revision_extrinsic_metadata: | if not p_info.revision_extrinsic_metadata: | ||||
| # If this package loader doesn't write metadata, no need to require | # If this package loader doesn't write metadata, no need to require | ||||
| # an implementation for get_metadata_authority. | # an implementation for get_metadata_authority. | ||||
| return [] | return [] | ||||
| ▲ Show 20 Lines • Show All 53 Lines • Show Last 20 Lines | |||||