Differential D4698 Diff 18681 swh/storage/postgresql/storage.py

Changeset View

Standalone View

swh/storage/postgresql/storage.py

# Copyright (C) 2015-2020 The Software Heritage developers		# Copyright (C) 2015-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import base64		import base64
from collections import defaultdict		from collections import defaultdict
import contextlib		import contextlib
from contextlib import contextmanager		from contextlib import contextmanager
Show All 9 Lines
from swh.core.api.serializers import msgpack_dumps, msgpack_loads		from swh.core.api.serializers import msgpack_dumps, msgpack_loads
from swh.core.db.common import db_transaction, db_transaction_generator		from swh.core.db.common import db_transaction, db_transaction_generator
from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex		from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex
from swh.model.identifiers import ExtendedObjectType, ExtendedSWHID		from swh.model.identifiers import ExtendedObjectType, ExtendedSWHID
from swh.model.model import (		from swh.model.model import (
SHA1_SIZE,		SHA1_SIZE,
Content,		Content,
Directory,		Directory,
		ExtID,
MetadataAuthority,		MetadataAuthority,
MetadataAuthorityType,		MetadataAuthorityType,
MetadataFetcher,		MetadataFetcher,
Origin,		Origin,
OriginVisit,		OriginVisit,
OriginVisitStatus,		OriginVisitStatus,
RawExtrinsicMetadata,		RawExtrinsicMetadata,
Release,		Release,
▲ Show 20 Lines • Show All 594 Lines • ▼ Show 20 Lines	) -> Iterable[Optional[Tuple[Sha1Git, Tuple[Sha1Git, ...]]]]:
yield from db.revision_shortlog(revisions, limit, cur)		yield from db.revision_shortlog(revisions, limit, cur)

@timed		@timed
@db_transaction()		@db_transaction()
def revision_get_random(self, db=None, cur=None) -> Sha1Git:		def revision_get_random(self, db=None, cur=None) -> Sha1Git:
return db.revision_get_random(cur)		return db.revision_get_random(cur)

@timed		@timed
		@db_transaction()
		def extid_get_from_extid(
		self, id_type: str, ids: List[bytes], db=None, cur=None
		) -> List[Optional[ExtID]]:
		"""Get ExtID objects from external IDs

		Args:
		id_type: type the external identifiers the SWH objects comes from
		ids: list of external IDs

		Returns:
		list of ExtID objects (if the ext ID is known, None otherwise)

		"""
		extids = []
		for row in db.extid_get_from_extid_list(id_type, ids, cur):
		extids.append(
		ExtID.from_dict(dict(zip(db.extid_cols, row)))
		if row[0] is not None
		else None
		)
		return extids

		@timed
		@db_transaction()
		def extid_get_from_target(
		self, target_type: ExtendedObjectType, ids: List[Sha1Git], db=None, cur=None
		) -> List[Optional[ExtID]]:
		"""Get ExtID objects from target IDs

		Args:
		target_type: type the SWH object
		ids: list of SWH IDs

		Returns:
		list of ExtID objects (if the SWH ID is known, None otherwise)

		"""
		extids = []
		for row in db.extid_get_from_swhid_list(target_type.value, ids, cur):
		extids.append(
		ExtID.from_dict(dict(zip(db.extid_cols, row)))
		if row[0] is not None
		else None
		)
		return extids

		@timed
		@db_transaction()
		def extid_add(self, ids: List[ExtID], db=None, cur=None) -> Dict[str, int]:
		"""Add a series of ExtID objects

		Args:
		ids: list of ExtID objects

		Returns:
		Summary dict of keys with associated count as values

		extid:add: New ExtID objects actually stored in db
		"""
		extid = [extid.to_dict() for extid in ids]
		db.mktemp("extid", cur)

		db.copy_to(extid, "tmp_extid", db.extid_cols, cur)

		# move metadata in place
		db.extid_add_from_temp(cur)

		return {"extid:add": len(extid)}

		@timed
@process_metrics		@process_metrics
@db_transaction()		@db_transaction()
def release_add(self, releases: List[Release], db=None, cur=None) -> Dict:		def release_add(self, releases: List[Release], db=None, cur=None) -> Dict:
summary = {"release:add": 0}		summary = {"release:add": 0}

release_ids = set(release.id for release in releases)		release_ids = set(release.id for release in releases)
releases_missing = set(self.release_missing(release_ids, db=db, cur=cur))		releases_missing = set(self.release_missing(release_ids, db=db, cur=cur))

▲ Show 20 Lines • Show All 759 Lines • Show Last 20 Lines