Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/backfill.py
| Show All 32 Lines | |||||
| ) | ) | ||||
| from swh.model.swhids import ExtendedObjectType | from swh.model.swhids import ExtendedObjectType | ||||
| from swh.storage.postgresql.converters import ( | from swh.storage.postgresql.converters import ( | ||||
| db_to_extid, | db_to_extid, | ||||
| db_to_raw_extrinsic_metadata, | db_to_raw_extrinsic_metadata, | ||||
| db_to_release, | db_to_release, | ||||
| db_to_revision, | db_to_revision, | ||||
| ) | ) | ||||
| from swh.storage.replay import object_converter_fn | from swh.storage.replay import OBJECT_CONVERTERS | ||||
| from swh.storage.writer import JournalWriter | from swh.storage.writer import JournalWriter | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| PARTITION_KEY = { | PARTITION_KEY = { | ||||
| "content": "sha1", | "content": "sha1", | ||||
| "skipped_content": "sha1", | "skipped_content": "sha1", | ||||
| "directory": "id", | "directory": "id", | ||||
| ▲ Show 20 Lines • Show All 486 Lines • ▼ Show 20 Lines | with db.cursor() as cursor: | ||||
| logger.debug("Fetching data for table %s", obj_type) | logger.debug("Fetching data for table %s", obj_type) | ||||
| logger.debug("query: %s %s", query, where_args) | logger.debug("query: %s %s", query, where_args) | ||||
| cursor.execute(query, where_args) | cursor.execute(query, where_args) | ||||
| for row in cursor: | for row in cursor: | ||||
| record = dict(zip(column_aliases, row)) | record = dict(zip(column_aliases, row)) | ||||
| if converter: | if converter: | ||||
| record = converter(db, record) | record = converter(db, record) | ||||
| else: | else: | ||||
| record = object_converter_fn[obj_type](record) | record = OBJECT_CONVERTERS[obj_type](record) | ||||
| logger.debug("record: %s", record) | logger.debug("record: %s", record) | ||||
| yield record | yield record | ||||
| def _format_range_bound(bound): | def _format_range_bound(bound): | ||||
| if isinstance(bound, bytes): | if isinstance(bound, bytes): | ||||
| return bound.hex() | return bound.hex() | ||||
| ▲ Show 20 Lines • Show All 106 Lines • Show Last 20 Lines | |||||