diff --git a/swh/storage/buffer.py b/swh/storage/buffer.py --- a/swh/storage/buffer.py +++ b/swh/storage/buffer.py @@ -4,12 +4,20 @@ # See top-level LICENSE file for more information from functools import partial -from typing import Dict, Iterable, Mapping, Sequence, Tuple +from typing import Dict, Iterable, List, Mapping, Sequence, Tuple from typing_extensions import Literal from swh.core.utils import grouper -from swh.model.model import BaseModel, Content, SkippedContent +from swh.model.model import ( + BaseModel, + Content, + MetadataAuthority, + MetadataFetcher, + Origin, + OriginVisitStatus, + SkippedContent, +) from swh.storage import get_storage from swh.storage.interface import StorageInterface @@ -21,6 +29,11 @@ "release", "snapshot", "extid", + "origin", + "origin_visit_status", + "raw_extrinsic_metadata", + "metadata_fetcher", + "metadata_authority", ] OBJECT_TYPES: Tuple[LObjectType, ...] = ( "content", @@ -30,6 +43,11 @@ "release", "snapshot", "extid", + "origin", + "origin_visit_status", + "raw_extrinsic_metadata", + "metadata_fetcher", + "metadata_authority", ) DEFAULT_BUFFER_THRESHOLDS: Dict[str, int] = { @@ -41,6 +59,11 @@ "release": 100000, "snapshot": 25000, "extid": 10000, + "origin": 100000, + "origin_visit_status": 10000, + "raw_extrinsic_metadata": 10000, + "metadata_fetcher": 100000, + "metadata_authority": 100000, } @@ -119,6 +142,30 @@ keys=["sha1", "sha1_git", "sha256", "blake2s256"], ) + def origin_add(self, origins: List[Origin]) -> Dict[str, int]: + return self.object_add(origins, object_type="origin", keys=["url"],) + + def origin_visit_status_add( + self, visit_statuses: List[OriginVisitStatus] + ) -> Dict[str, int]: + return self.object_add( + visit_statuses, + object_type="origin_visit_status", + keys=["origin", "visit", "date"], + ) + + def metadata_authority_add( + self, authorities: List[MetadataAuthority] + ) -> Dict[str, int]: + return self.object_add( + authorities, object_type="metadata_authority", keys=["type", "url"], + ) + + def metadata_fetcher_add(self, fetchers: List[MetadataFetcher]) -> Dict[str, int]: + return self.object_add( + fetchers, object_type="metadata_fetcher", keys=["name", "version"], + ) + def object_add( self, objects: Sequence[BaseModel], diff --git a/swh/storage/filter.py b/swh/storage/filter.py --- a/swh/storage/filter.py +++ b/swh/storage/filter.py @@ -27,8 +27,6 @@ """ - object_types = ["content", "skipped_content", "directory", "revision"] - def __init__(self, storage): self.storage: StorageInterface = get_storage(**storage) diff --git a/swh/storage/validate.py b/swh/storage/validate.py --- a/swh/storage/validate.py +++ b/swh/storage/validate.py @@ -7,7 +7,17 @@ from typing import Dict, Iterable, List from swh.model.hashutil import MultiHash, hash_to_bytes, hash_to_hex -from swh.model.model import Content, Directory, Release, Revision, Snapshot +from swh.model.model import ( + Content, + Directory, + MetadataAuthority, + MetadataFetcher, + OriginVisitStatus, + RawExtrinsicMetadata, + Release, + Revision, + Snapshot, +) from swh.storage import get_storage from swh.storage.exc import StorageArgumentException from swh.storage.interface import StorageInterface @@ -69,3 +79,25 @@ def snapshot_add(self, snapshots: List[Snapshot]) -> Dict[str, int]: self._check_hashes(snapshots) return self.storage.snapshot_add(snapshots) + + def origin_visit_status_add( + self, visit_statuses: List[OriginVisitStatus], db=None, cur=None, + ) -> Dict[str, int]: + self._check_hashes(visit_statuses) + return self.storage.origin_visit_status_add(visit_statuses) + + def raw_extrinsic_metadata_add( + self, metadata: List[RawExtrinsicMetadata] + ) -> Dict[str, int]: + self._check_hashes(metadata) + return self.storage.raw_extrinsic_metadata_add(metadata) + + def metadata_fetcher_add(self, fetchers: List[MetadataFetcher]) -> Dict[str, int]: + self._check_hashes(fetchers) + return self.storage.metadata_fetcher_add(fetchers) + + def metadata_authority_add( + self, authorities: List[MetadataAuthority] + ) -> Dict[str, int]: + self._check_hashes(authorities) + return self.storage.metadata_authority_add(authorities)