Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/backfill.py
| Show All 18 Lines | |||||
| from typing import Any, Callable, Dict, Optional | from typing import Any, Callable, Dict, Optional | ||||
| from swh.core.db import BaseDb | from swh.core.db import BaseDb | ||||
| from swh.model.identifiers import ExtendedObjectType | from swh.model.identifiers import ExtendedObjectType | ||||
| from swh.model.model import ( | from swh.model.model import ( | ||||
| BaseModel, | BaseModel, | ||||
| Directory, | Directory, | ||||
| DirectoryEntry, | DirectoryEntry, | ||||
| ExtID, | |||||
| RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
| Release, | Release, | ||||
| Revision, | Revision, | ||||
| Snapshot, | Snapshot, | ||||
| SnapshotBranch, | SnapshotBranch, | ||||
| TargetType, | TargetType, | ||||
| ) | ) | ||||
| from swh.storage.postgresql.converters import ( | from swh.storage.postgresql.converters import ( | ||||
| db_to_extid, | |||||
| db_to_raw_extrinsic_metadata, | db_to_raw_extrinsic_metadata, | ||||
| db_to_release, | db_to_release, | ||||
| db_to_revision, | db_to_revision, | ||||
| ) | ) | ||||
| from swh.storage.replay import object_converter_fn | from swh.storage.replay import object_converter_fn | ||||
| from swh.storage.writer import JournalWriter | from swh.storage.writer import JournalWriter | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| PARTITION_KEY = { | PARTITION_KEY = { | ||||
| "content": "sha1", | "content": "sha1", | ||||
| "skipped_content": "sha1", | "skipped_content": "sha1", | ||||
| "directory": "id", | "directory": "id", | ||||
| "extid": "target", | |||||
| "metadata_authority": "type, url", | "metadata_authority": "type, url", | ||||
| "metadata_fetcher": "name, version", | "metadata_fetcher": "name, version", | ||||
| "raw_extrinsic_metadata": "target", | "raw_extrinsic_metadata": "target", | ||||
| "revision": "revision.id", | "revision": "revision.id", | ||||
| "release": "release.id", | "release": "release.id", | ||||
| "snapshot": "id", | "snapshot": "id", | ||||
| "origin": "id", | "origin": "id", | ||||
| "origin_visit": "origin_visit.origin", | "origin_visit": "origin_visit.origin", | ||||
| Show All 16 Lines | "skipped_content": [ | ||||
| "sha256", | "sha256", | ||||
| "blake2s256", | "blake2s256", | ||||
| "length", | "length", | ||||
| "ctime", | "ctime", | ||||
| "status", | "status", | ||||
| "reason", | "reason", | ||||
| ], | ], | ||||
| "directory": ["id", "dir_entries", "file_entries", "rev_entries"], | "directory": ["id", "dir_entries", "file_entries", "rev_entries"], | ||||
| "extid": ["extid_type", "extid", "target_type", "target"], | |||||
| "metadata_authority": ["type", "url", "metadata",], | "metadata_authority": ["type", "url", "metadata",], | ||||
| "metadata_fetcher": ["name", "version", "metadata",], | "metadata_fetcher": ["name", "version", "metadata",], | ||||
| "origin": ["url"], | |||||
| "origin_visit": ["visit", "type", ("origin.url", "origin"), "date",], | |||||
| "origin_visit_status": [ | |||||
| ("origin_visit_status.visit", "visit"), | |||||
| ("origin.url", "origin"), | |||||
| ("origin_visit_status.date", "date"), | |||||
| "type", | |||||
| "snapshot", | |||||
| "status", | |||||
| "metadata", | |||||
| ], | |||||
| "raw_extrinsic_metadata": [ | "raw_extrinsic_metadata": [ | ||||
| "raw_extrinsic_metadata.type", | "raw_extrinsic_metadata.type", | ||||
| "raw_extrinsic_metadata.target", | "raw_extrinsic_metadata.target", | ||||
| "metadata_authority.type", | "metadata_authority.type", | ||||
| "metadata_authority.url", | "metadata_authority.url", | ||||
| "metadata_fetcher.name", | "metadata_fetcher.name", | ||||
| "metadata_fetcher.version", | "metadata_fetcher.version", | ||||
| "discovery_date", | "discovery_date", | ||||
| ▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | "release": [ | ||||
| "target", | "target", | ||||
| "target_type", | "target_type", | ||||
| ("a.id", "author_id"), | ("a.id", "author_id"), | ||||
| ("a.name", "author_name"), | ("a.name", "author_name"), | ||||
| ("a.email", "author_email"), | ("a.email", "author_email"), | ||||
| ("a.fullname", "author_fullname"), | ("a.fullname", "author_fullname"), | ||||
| ], | ], | ||||
| "snapshot": ["id", "object_id"], | "snapshot": ["id", "object_id"], | ||||
| "origin": ["url"], | |||||
| "origin_visit": ["visit", "type", ("origin.url", "origin"), "date",], | |||||
| "origin_visit_status": [ | |||||
| ("origin_visit_status.visit", "visit"), | |||||
| ("origin.url", "origin"), | |||||
| ("origin_visit_status.date", "date"), | |||||
| "type", | |||||
| "snapshot", | |||||
| "status", | |||||
| "metadata", | |||||
| ], | |||||
| } | } | ||||
| JOINS = { | JOINS = { | ||||
| "release": ["person a on release.author=a.id"], | "release": ["person a on release.author=a.id"], | ||||
| "revision": [ | "revision": [ | ||||
| "person a on revision.author=a.id", | "person a on revision.author=a.id", | ||||
| "person c on revision.committer=c.id", | "person c on revision.committer=c.id", | ||||
| ▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | with db.cursor() as cur: | ||||
| entries.append(entry) | entries.append(entry) | ||||
| return Directory(id=directory_d["id"], entries=tuple(entries),) | return Directory(id=directory_d["id"], entries=tuple(entries),) | ||||
| def raw_extrinsic_metadata_converter( | def raw_extrinsic_metadata_converter( | ||||
| db: BaseDb, metadata: Dict[str, Any] | db: BaseDb, metadata: Dict[str, Any] | ||||
| ) -> RawExtrinsicMetadata: | ) -> RawExtrinsicMetadata: | ||||
| """Convert revision from the flat representation to swh model | """Convert a raw extrinsic metadata from the flat representation to swh model | ||||
| compatible objects. | compatible objects. | ||||
| """ | """ | ||||
| return db_to_raw_extrinsic_metadata(metadata) | return db_to_raw_extrinsic_metadata(metadata) | ||||
| def extid_converter(db: BaseDb, extid: Dict[str, Any]) -> ExtID: | |||||
| """Convert an extid from the flat representation to swh model | |||||
| compatible objects. | |||||
| """ | |||||
| return db_to_extid(extid) | |||||
| def revision_converter(db: BaseDb, revision_d: Dict[str, Any]) -> Revision: | def revision_converter(db: BaseDb, revision_d: Dict[str, Any]) -> Revision: | ||||
| """Convert revision from the flat representation to swh model | """Convert revision from the flat representation to swh model | ||||
| compatible objects. | compatible objects. | ||||
| """ | """ | ||||
| revision = db_to_revision(revision_d) | revision = db_to_revision(revision_d) | ||||
| assert revision is not None, revision_d["id"] | assert revision is not None, revision_d["id"] | ||||
| return revision | return revision | ||||
| Show All 37 Lines | with db.cursor() as cur: | ||||
| branch = None | branch = None | ||||
| branches[name] = branch | branches[name] = branch | ||||
| return Snapshot(id=snapshot_d["id"], branches=branches,) | return Snapshot(id=snapshot_d["id"], branches=branches,) | ||||
| CONVERTERS: Dict[str, Callable[[BaseDb, Dict[str, Any]], BaseModel]] = { | CONVERTERS: Dict[str, Callable[[BaseDb, Dict[str, Any]], BaseModel]] = { | ||||
| "directory": directory_converter, | "directory": directory_converter, | ||||
| "extid": extid_converter, | |||||
| "raw_extrinsic_metadata": raw_extrinsic_metadata_converter, | "raw_extrinsic_metadata": raw_extrinsic_metadata_converter, | ||||
| "revision": revision_converter, | "revision": revision_converter, | ||||
| "release": release_converter, | "release": release_converter, | ||||
| "snapshot": snapshot_converter, | "snapshot": snapshot_converter, | ||||
| } | } | ||||
| def object_to_offset(object_id, numbits): | def object_to_offset(object_id, numbits): | ||||
| ▲ Show 20 Lines • Show All 145 Lines • ▼ Show 20 Lines | for start in range(start, end, block_size): | ||||
| else: | else: | ||||
| yield start, start + block_size | yield start, start + block_size | ||||
| RANGE_GENERATORS = { | RANGE_GENERATORS = { | ||||
| "content": lambda start, end: byte_ranges(24, start, end), | "content": lambda start, end: byte_ranges(24, start, end), | ||||
| "skipped_content": lambda start, end: [(None, None)], | "skipped_content": lambda start, end: [(None, None)], | ||||
| "directory": lambda start, end: byte_ranges(24, start, end), | "directory": lambda start, end: byte_ranges(24, start, end), | ||||
| "extid": lambda start, end: byte_ranges(24, start, end), | |||||
| "revision": lambda start, end: byte_ranges(24, start, end), | "revision": lambda start, end: byte_ranges(24, start, end), | ||||
| "release": lambda start, end: byte_ranges(16, start, end), | "release": lambda start, end: byte_ranges(16, start, end), | ||||
| "raw_extrinsic_metadata": raw_extrinsic_metadata_target_ranges, | "raw_extrinsic_metadata": raw_extrinsic_metadata_target_ranges, | ||||
| "snapshot": lambda start, end: byte_ranges(16, start, end), | "snapshot": lambda start, end: byte_ranges(16, start, end), | ||||
| "origin": integer_ranges, | "origin": integer_ranges, | ||||
| "origin_visit": integer_ranges, | "origin_visit": integer_ranges, | ||||
| "origin_visit_status": integer_ranges, | "origin_visit_status": integer_ranges, | ||||
| } | } | ||||
| ▲ Show 20 Lines • Show All 130 Lines • ▼ Show 20 Lines | def parse_arguments(self, object_type, start_object, end_object): | ||||
| Returns: | Returns: | ||||
| Parsed start and end object ids | Parsed start and end object ids | ||||
| """ | """ | ||||
| if object_type not in COLUMNS: | if object_type not in COLUMNS: | ||||
| raise ValueError( | raise ValueError( | ||||
| "Object type %s is not supported. " | "Object type %s is not supported. " | ||||
| "The only possible values are %s" | "The only possible values are %s" | ||||
| % (object_type, ", ".join(COLUMNS.keys())) | % (object_type, ", ".join(sorted(COLUMNS.keys()))) | ||||
| ) | ) | ||||
| if object_type in ["origin", "origin_visit", "origin_visit_status"]: | if object_type in ["origin", "origin_visit", "origin_visit_status"]: | ||||
| if start_object: | if start_object: | ||||
| start_object = int(start_object) | start_object = int(start_object) | ||||
| else: | else: | ||||
| start_object = 0 | start_object = 0 | ||||
| if end_object: | if end_object: | ||||
| ▲ Show 20 Lines • Show All 42 Lines • Show Last 20 Lines | |||||