Differential D8905 Diff 32095 swh/indexer/metadata.py

Changeset View

Standalone View

swh/indexer/metadata.py

	# Copyright (C) 2017-2022 The Software Heritage developers			# Copyright (C) 2017-2022 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution			# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

	from copy import deepcopy			from copy import deepcopy
	import hashlib			import hashlib
	import itertools
	import logging			import logging
	import time			import time
	from typing import (			from typing import (
	Any,			Any,
	Callable,			Callable,
	Dict,			Dict,
	Iterable,			Iterable,
	Iterator,			Iterator,
	▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines
	):			):
	def process_journal_objects(self, objects: ObjectsDict) -> Dict:			def process_journal_objects(self, objects: ObjectsDict) -> Dict:
	summary: Dict[str, Any] = {"status": "uneventful"}			summary: Dict[str, Any] = {"status": "uneventful"}
	try:			try:
	results = {}			results = {}
	for item in objects.get("raw_extrinsic_metadata", []):			for item in objects.get("raw_extrinsic_metadata", []):
	remd = RawExtrinsicMetadata.from_dict(item)			remd = RawExtrinsicMetadata.from_dict(item)
	sentry_sdk.set_tag("swh-indexer-remd-swhid", str(remd.swhid()))			sentry_sdk.set_tag("swh-indexer-remd-swhid", str(remd.swhid()))
	results[remd.target] = self.index(remd.id, data=remd)			for result in self.index(remd.id, data=remd):
				results[result.id] = result
	except Exception:			except Exception:
	if not self.catch_exceptions:			if not self.catch_exceptions:
	raise			raise
	summary["status"] = "failed"			summary["status"] = "failed"
	return summary			return summary

	self.results = list(itertools.chain.from_iterable(results.values()))			self.results = list(results.values())
	summary_persist = self.persist_index_computations(self.results)			summary_persist = self.persist_index_computations(self.results)
	if summary_persist:			if summary_persist:
	for value in summary_persist.values():			for value in summary_persist.values():
	if value > 0:			if value > 0:
	summary["status"] = "eventful"			summary["status"] = "eventful"
	summary.update(summary_persist)			summary.update(summary_persist)
	return summary			return summary

	▲ Show 20 Lines • Show All 465 Lines • Show Last 20 Lines