Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/storage.py
Show All 19 Lines | |||||
) | ) | ||||
import attr | import attr | ||||
import psycopg2 | import psycopg2 | ||||
import psycopg2.pool | import psycopg2.pool | ||||
import psycopg2.errors | import psycopg2.errors | ||||
from swh.core.api.serializers import msgpack_loads, msgpack_dumps | from swh.core.api.serializers import msgpack_loads, msgpack_dumps | ||||
from swh.model.identifiers import parse_swhid, SWHID | from swh.model.identifiers import SWHID | ||||
from swh.model.model import ( | from swh.model.model import ( | ||||
Content, | Content, | ||||
Directory, | Directory, | ||||
Origin, | Origin, | ||||
OriginVisit, | OriginVisit, | ||||
OriginVisitStatus, | OriginVisitStatus, | ||||
Revision, | Revision, | ||||
Release, | Release, | ||||
▲ Show 20 Lines • Show All 1,235 Lines • ▼ Show 20 Lines | ) -> Dict[str, Union[Optional[bytes], List[RawExtrinsicMetadata]]]: | ||||
after_time, | after_time, | ||||
after_fetcher, | after_fetcher, | ||||
limit + 1, | limit + 1, | ||||
cur, | cur, | ||||
) | ) | ||||
rows = [dict(zip(db.raw_extrinsic_metadata_get_cols, row)) for row in rows] | rows = [dict(zip(db.raw_extrinsic_metadata_get_cols, row)) for row in rows] | ||||
results = [] | results = [] | ||||
for row in rows: | for row in rows: | ||||
row = row.copy() | |||||
row.pop("metadata_fetcher.id") | |||||
assert str(id) == row["raw_extrinsic_metadata.id"] | assert str(id) == row["raw_extrinsic_metadata.id"] | ||||
results.append(converters.db_to_raw_extrinsic_metadata(row)) | |||||
result = RawExtrinsicMetadata( | |||||
type=MetadataTargetType(row["raw_extrinsic_metadata.type"]), | |||||
id=id, | |||||
authority=MetadataAuthority( | |||||
type=MetadataAuthorityType(row["metadata_authority.type"]), | |||||
url=row["metadata_authority.url"], | |||||
), | |||||
fetcher=MetadataFetcher( | |||||
name=row["metadata_fetcher.name"], | |||||
version=row["metadata_fetcher.version"], | |||||
), | |||||
discovery_date=row["discovery_date"], | |||||
format=row["format"], | |||||
metadata=row["raw_extrinsic_metadata.metadata"], | |||||
origin=row["origin"], | |||||
visit=row["visit"], | |||||
snapshot=map_optional(parse_swhid, row["snapshot"]), | |||||
release=map_optional(parse_swhid, row["release"]), | |||||
revision=map_optional(parse_swhid, row["revision"]), | |||||
path=row["path"], | |||||
directory=map_optional(parse_swhid, row["directory"]), | |||||
) | |||||
results.append(result) | |||||
if len(results) > limit: | if len(results) > limit: | ||||
results.pop() | results.pop() | ||||
assert len(results) == limit | assert len(results) == limit | ||||
last_returned_row = rows[-2] # rows[-1] corresponds to the popped result | last_returned_row = rows[-2] # rows[-1] corresponds to the popped result | ||||
next_page_token: Optional[bytes] = msgpack_dumps( | next_page_token: Optional[bytes] = msgpack_dumps( | ||||
( | ( | ||||
last_returned_row["discovery_date"], | last_returned_row["discovery_date"], | ||||
▲ Show 20 Lines • Show All 106 Lines • Show Last 20 Lines |