Differential D4698 Diff 16661 swh/storage/postgresql/storage.py

Changeset View

Standalone View

swh/storage/postgresql/storage.py

Show All 22 Lines
from swh.model.model import (		from swh.model.model import (
SHA1_SIZE,		SHA1_SIZE,
Content,		Content,
Directory,		Directory,
MetadataAuthority,		MetadataAuthority,
MetadataAuthorityType,		MetadataAuthorityType,
MetadataFetcher,		MetadataFetcher,
MetadataTargetType,		MetadataTargetType,
		ObjectType,
Origin,		Origin,
OriginVisit,		OriginVisit,
OriginVisitStatus,		OriginVisitStatus,
RawExtrinsicMetadata,		RawExtrinsicMetadata,
Release,		Release,
Revision,		Revision,
Sha1,		Sha1,
Sha1Git,		Sha1Git,
SkippedContent,		SkippedContent,
Snapshot,		Snapshot,
SnapshotBranch,		SnapshotBranch,
TargetType,		TargetType,
)		)
		from swh.storage.common import revision_to_vcsid
from swh.storage.exc import HashCollision, StorageArgumentException, StorageDBError		from swh.storage.exc import HashCollision, StorageArgumentException, StorageDBError
from swh.storage.interface import (		from swh.storage.interface import (
VISIT_STATUSES,		VISIT_STATUSES,
ListOrder,		ListOrder,
PagedResult,		PagedResult,
PartialBranches,		PartialBranches,
)		)
from swh.storage.metrics import process_metrics, send_metric, timed		from swh.storage.metrics import process_metrics, send_metric, timed
▲ Show 20 Lines • Show All 528 Lines • ▼ Show 20 Lines	def revision_add(self, revisions: List[Revision], db=None, cur=None) -> Dict:

db.copy_to(		db.copy_to(
parents_filtered,		parents_filtered,
"revision_history",		"revision_history",
["id", "parent_id", "parent_rank"],		["id", "parent_id", "parent_rank"],
cur,		cur,
)		)

		# insert newly added revision IDs to the vcsid table
		vcsid = [
		{
		"vcs": revision.type.value,
		"vcs_id": revision_to_vcsid(revision),
		"swhid_type": ObjectType.REVISION.value,
		"swhid_value": revision.id,
		}
		for revision in revisions_filtered
		]
		db.mktemp("vcsid", cur)

		db.copy_to(vcsid, "tmp_vcsid", db.vcsid_cols, cur)

		# move metadata in place
		db.vcsid_add_from_temp(cur)

return {"revision:add": len(revisions_missing)}		return {"revision:add": len(revisions_missing)}

@timed		@timed
@db_transaction_generator()		@db_transaction_generator()
def revision_missing(		def revision_missing(
self, revisions: List[Sha1Git], db=None, cur=None		self, revisions: List[Sha1Git], db=None, cur=None
) -> Iterable[Sha1Git]:		) -> Iterable[Sha1Git]:
if not revisions:		if not revisions:
Show All 10 Lines	) -> List[Optional[Revision]]:
revisions = []		revisions = []
for line in db.revision_get_from_list(revision_ids, cur):		for line in db.revision_get_from_list(revision_ids, cur):
revision = converters.db_to_revision(dict(zip(db.revision_get_cols, line)))		revision = converters.db_to_revision(dict(zip(db.revision_get_cols, line)))
revisions.append(revision)		revisions.append(revision)

return revisions		return revisions

@timed		@timed
		@db_transaction(statement_timeout=1000)
		def revision_id_from_vcs(
		self, vcs_type: str, vcs_ids: List[bytes], db=None, cur=None
		) -> List[Optional[Sha1Git]]:
		rev_ids = []
		for line in db.swhid_get_from_vcsid_list(vcs_type, vcs_ids, cur):
		rev_ids.append(line[3])

		return rev_ids

		@timed
@db_transaction_generator(statement_timeout=2000)		@db_transaction_generator(statement_timeout=2000)
def revision_log(		def revision_log(
self, revisions: List[Sha1Git], limit: Optional[int] = None, db=None, cur=None		self, revisions: List[Sha1Git], limit: Optional[int] = None, db=None, cur=None
) -> Iterable[Optional[Dict[str, Any]]]:		) -> Iterable[Optional[Dict[str, Any]]]:
for line in db.revision_log(revisions, limit, cur):		for line in db.revision_log(revisions, limit, cur):
data = converters.db_to_revision(dict(zip(db.revision_get_cols, line)))		data = converters.db_to_revision(dict(zip(db.revision_get_cols, line)))
if not data:		if not data:
yield None		yield None
▲ Show 20 Lines • Show All 799 Lines • Show Last 20 Lines