Differential D4698 Diff 17169 swh/storage/postgresql/storage.py

Changeset View

Standalone View

swh/storage/postgresql/storage.py

Show All 18 Lines
from swh.core.api.serializers import msgpack_dumps, msgpack_loads		from swh.core.api.serializers import msgpack_dumps, msgpack_loads
from swh.core.db.common import db_transaction, db_transaction_generator		from swh.core.db.common import db_transaction, db_transaction_generator
from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex		from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex
from swh.model.identifiers import SWHID		from swh.model.identifiers import SWHID
from swh.model.model import (		from swh.model.model import (
SHA1_SIZE,		SHA1_SIZE,
Content,		Content,
Directory,		Directory,
		ExtID,
MetadataAuthority,		MetadataAuthority,
MetadataAuthorityType,		MetadataAuthorityType,
MetadataFetcher,		MetadataFetcher,
MetadataTargetType,		MetadataTargetType,
		ObjectType,
Origin,		Origin,
OriginVisit,		OriginVisit,
OriginVisitStatus,		OriginVisitStatus,
RawExtrinsicMetadata,		RawExtrinsicMetadata,
Release,		Release,
Revision,		Revision,
Sha1,		Sha1,
Sha1Git,		Sha1Git,
▲ Show 20 Lines • Show All 591 Lines • ▼ Show 20 Lines	) -> Iterable[Optional[Tuple[Sha1Git, Tuple[Sha1Git, ...]]]]:
yield from db.revision_shortlog(revisions, limit, cur)		yield from db.revision_shortlog(revisions, limit, cur)

@timed		@timed
@db_transaction()		@db_transaction()
def revision_get_random(self, db=None, cur=None) -> Sha1Git:		def revision_get_random(self, db=None, cur=None) -> Sha1Git:
return db.revision_get_random(cur)		return db.revision_get_random(cur)

@timed		@timed
		@db_transaction()
		def extid_get_from_extid(
		self, id_type: str, ids: List[bytes], db=None, cur=None
		) -> List[Optional[ExtID]]:
		"""Get ExtID objects from external IDs

		Args:
		id_type: type the external identifiers the SWH objects comes from
		ids: list of external IDs

		Returns:
		list of ExtID objects (if the ext ID is known, None otherwise)

		"""
		extids = []
		for row in db.extid_get_from_extid_list(id_type, ids, cur):
		extids.append(
		ExtID.from_dict(dict(zip(db.extid_cols, row)))
		if row[0] is not None
		else None
		)
		return extids

		@timed
		@db_transaction()
		def extid_get_from_target(
		self, target_type: ObjectType, ids: List[Sha1Git], db=None, cur=None
		) -> List[Optional[ExtID]]:
		"""Get ExtID objects from target IDs

		Args:
		target_type: type the SWH object
		ids: list of SWH IDs

		Returns:
		list of ExtID objects (if the SWH ID is known, None otherwise)

		"""
		extids = []
		for row in db.extid_get_from_swhid_list(target_type.value, ids, cur):
		extids.append(
		ExtID.from_dict(dict(zip(db.extid_cols, row)))
		if row[0] is not None
		else None
		)
		return extids

		@timed
		@db_transaction()
		def extid_add(self, ids: List[ExtID], db=None, cur=None) -> Dict[str, int]:
		"""Add a series of ExtID objects

		Args:
		ids: list of ExtID objects

		Returns:
		Summary dict of keys with associated count as values

		extid:add: New ExtID objects actually stored in db
		"""
		extid = [extid.to_dict() for extid in ids]
		db.mktemp("extid", cur)

		db.copy_to(extid, "tmp_extid", db.extid_cols, cur)

		# move metadata in place
		db.extid_add_from_temp(cur)

		return {"extid:add": len(extid)}

		@timed
@process_metrics		@process_metrics
@db_transaction()		@db_transaction()
def release_add(self, releases: List[Release], db=None, cur=None) -> Dict:		def release_add(self, releases: List[Release], db=None, cur=None) -> Dict:
summary = {"release:add": 0}		summary = {"release:add": 0}

release_ids = set(release.id for release in releases)		release_ids = set(release.id for release in releases)
releases_missing = set(self.release_missing(release_ids, db=db, cur=cur))		releases_missing = set(self.release_missing(release_ids, db=db, cur=cur))

▲ Show 20 Lines • Show All 775 Lines • Show Last 20 Lines