Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/backfill.py
Show All 32 Lines | |||||
) | ) | ||||
from swh.model.swhids import ExtendedObjectType | from swh.model.swhids import ExtendedObjectType | ||||
from swh.storage.postgresql.converters import ( | from swh.storage.postgresql.converters import ( | ||||
db_to_extid, | db_to_extid, | ||||
db_to_raw_extrinsic_metadata, | db_to_raw_extrinsic_metadata, | ||||
db_to_release, | db_to_release, | ||||
db_to_revision, | db_to_revision, | ||||
) | ) | ||||
from swh.storage.replay import object_converter_fn | from swh.storage.replay import OBJECT_CONVERTERS | ||||
from swh.storage.writer import JournalWriter | from swh.storage.writer import JournalWriter | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
PARTITION_KEY = { | PARTITION_KEY = { | ||||
"content": "sha1", | "content": "sha1", | ||||
"skipped_content": "sha1", | "skipped_content": "sha1", | ||||
"directory": "id", | "directory": "id", | ||||
▲ Show 20 Lines • Show All 486 Lines • ▼ Show 20 Lines | with db.cursor() as cursor: | ||||
logger.debug("Fetching data for table %s", obj_type) | logger.debug("Fetching data for table %s", obj_type) | ||||
logger.debug("query: %s %s", query, where_args) | logger.debug("query: %s %s", query, where_args) | ||||
cursor.execute(query, where_args) | cursor.execute(query, where_args) | ||||
for row in cursor: | for row in cursor: | ||||
record = dict(zip(column_aliases, row)) | record = dict(zip(column_aliases, row)) | ||||
if converter: | if converter: | ||||
record = converter(db, record) | record = converter(db, record) | ||||
else: | else: | ||||
record = object_converter_fn[obj_type](record) | record = OBJECT_CONVERTERS[obj_type](record) | ||||
logger.debug("record: %s", record) | logger.debug("record: %s", record) | ||||
yield record | yield record | ||||
def _format_range_bound(bound): | def _format_range_bound(bound): | ||||
if isinstance(bound, bytes): | if isinstance(bound, bytes): | ||||
return bound.hex() | return bound.hex() | ||||
▲ Show 20 Lines • Show All 106 Lines • Show Last 20 Lines |