Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/interface.py
Show All 33 Lines | |||||
class StorageInterface: | class StorageInterface: | ||||
@remote_api_endpoint("check_config") | @remote_api_endpoint("check_config") | ||||
def check_config(self, *, check_write): | def check_config(self, *, check_write): | ||||
"""Check that the storage is configured and ready to go.""" | """Check that the storage is configured and ready to go.""" | ||||
... | ... | ||||
@remote_api_endpoint("content/add") | @remote_api_endpoint("content/add") | ||||
def content_add(self, content: Iterable[Content]) -> Dict: | def content_add(self, content: List[Content]) -> Dict: | ||||
"""Add content blobs to the storage | """Add content blobs to the storage | ||||
Args: | Args: | ||||
contents (iterable): iterable of dictionaries representing | contents (iterable): iterable of dictionaries representing | ||||
individual pieces of content to add. Each dictionary has the | individual pieces of content to add. Each dictionary has the | ||||
following keys: | following keys: | ||||
- data (bytes): the actual content | - data (bytes): the actual content | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | def content_update(self, content, keys=[]): | ||||
keys (list): List of keys (str) whose values needs an update, e.g., | keys (list): List of keys (str) whose values needs an update, e.g., | ||||
new hash column | new hash column | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("content/add_metadata") | @remote_api_endpoint("content/add_metadata") | ||||
def content_add_metadata(self, content: Iterable[Content]) -> Dict: | def content_add_metadata(self, content: List[Content]) -> Dict: | ||||
"""Add content metadata to the storage (like `content_add`, but | """Add content metadata to the storage (like `content_add`, but | ||||
without inserting to the objstorage). | without inserting to the objstorage). | ||||
Args: | Args: | ||||
content (iterable): iterable of dictionaries representing | content (iterable): iterable of dictionaries representing | ||||
individual pieces of content to add. Each dictionary has the | individual pieces of content to add. Each dictionary has the | ||||
following keys: | following keys: | ||||
▲ Show 20 Lines • Show All 134 Lines • ▼ Show 20 Lines | def content_missing(self, content, key_hash="sha1"): | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("content/missing/sha1") | @remote_api_endpoint("content/missing/sha1") | ||||
def content_missing_per_sha1(self, contents): | def content_missing_per_sha1(self, contents): | ||||
"""List content missing from storage based only on sha1. | """List content missing from storage based only on sha1. | ||||
Args: | Args: | ||||
contents: Iterable of sha1 to check for absence. | contents: List of sha1 to check for absence. | ||||
Returns: | Returns: | ||||
iterable: missing ids | iterable: missing ids | ||||
Raises: | Raises: | ||||
TODO: an exception when we get a hash collision. | TODO: an exception when we get a hash collision. | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("content/missing/sha1_git") | @remote_api_endpoint("content/missing/sha1_git") | ||||
def content_missing_per_sha1_git(self, contents): | def content_missing_per_sha1_git(self, contents): | ||||
"""List content missing from storage based only on sha1_git. | """List content missing from storage based only on sha1_git. | ||||
Args: | Args: | ||||
contents (Iterable): An iterable of content id (sha1_git) | contents (List): An iterable of content id (sha1_git) | ||||
Yields: | Yields: | ||||
missing contents sha1_git | missing contents sha1_git | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("content/present") | @remote_api_endpoint("content/present") | ||||
def content_find(self, content): | def content_find(self, content): | ||||
Show All 20 Lines | def content_get_random(self): | ||||
"""Finds a random content id. | """Finds a random content id. | ||||
Returns: | Returns: | ||||
a sha1_git | a sha1_git | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("content/skipped/add") | @remote_api_endpoint("content/skipped/add") | ||||
def skipped_content_add(self, content: Iterable[SkippedContent]) -> Dict: | def skipped_content_add(self, content: List[SkippedContent]) -> Dict: | ||||
"""Add contents to the skipped_content list, which contains | """Add contents to the skipped_content list, which contains | ||||
(partial) information about content missing from the archive. | (partial) information about content missing from the archive. | ||||
Args: | Args: | ||||
contents (iterable): iterable of dictionaries representing | contents (iterable): iterable of dictionaries representing | ||||
individual pieces of content to add. Each dictionary has the | individual pieces of content to add. Each dictionary has the | ||||
following keys: | following keys: | ||||
Show All 34 Lines | def skipped_content_missing(self, contents): | ||||
Returns: | Returns: | ||||
iterable: missing signatures | iterable: missing signatures | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("directory/add") | @remote_api_endpoint("directory/add") | ||||
def directory_add(self, directories: Iterable[Directory]) -> Dict: | def directory_add(self, directories: List[Directory]) -> Dict: | ||||
"""Add directories to the storage | """Add directories to the storage | ||||
Args: | Args: | ||||
directories (iterable): iterable of dictionaries representing the | directories (iterable): iterable of dictionaries representing the | ||||
individual directories to add. Each dict has the following | individual directories to add. Each dict has the following | ||||
keys: | keys: | ||||
- id (sha1_git): the id of the directory to add | - id (sha1_git): the id of the directory to add | ||||
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | def directory_get_random(self): | ||||
"""Finds a random directory id. | """Finds a random directory id. | ||||
Returns: | Returns: | ||||
a sha1_git | a sha1_git | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("revision/add") | @remote_api_endpoint("revision/add") | ||||
def revision_add(self, revisions: Iterable[Revision]) -> Dict: | def revision_add(self, revisions: List[Revision]) -> Dict: | ||||
"""Add revisions to the storage | """Add revisions to the storage | ||||
Args: | Args: | ||||
revisions (Iterable[dict]): iterable of dictionaries representing | revisions (List[dict]): iterable of dictionaries representing | ||||
the individual revisions to add. Each dict has the following | the individual revisions to add. Each dict has the following | ||||
keys: | keys: | ||||
- **id** (:class:`sha1_git`): id of the revision to add | - **id** (:class:`sha1_git`): id of the revision to add | ||||
- **date** (:class:`dict`): date the revision was written | - **date** (:class:`dict`): date the revision was written | ||||
- **committer_date** (:class:`dict`): date the revision got | - **committer_date** (:class:`dict`): date the revision got | ||||
added to the origin | added to the origin | ||||
- **type** (one of 'git', 'tar'): type of the | - **type** (one of 'git', 'tar'): type of the | ||||
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines | def revision_get_random(self): | ||||
"""Finds a random revision id. | """Finds a random revision id. | ||||
Returns: | Returns: | ||||
a sha1_git | a sha1_git | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("release/add") | @remote_api_endpoint("release/add") | ||||
def release_add(self, releases: Iterable[Release]) -> Dict: | def release_add(self, releases: List[Release]) -> Dict: | ||||
"""Add releases to the storage | """Add releases to the storage | ||||
Args: | Args: | ||||
releases (Iterable[dict]): iterable of dictionaries representing | releases (List[dict]): iterable of dictionaries representing | ||||
the individual releases to add. Each dict has the following | the individual releases to add. Each dict has the following | ||||
keys: | keys: | ||||
- **id** (:class:`sha1_git`): id of the release to add | - **id** (:class:`sha1_git`): id of the release to add | ||||
- **revision** (:class:`sha1_git`): id of the revision the | - **revision** (:class:`sha1_git`): id of the revision the | ||||
release points to | release points to | ||||
- **date** (:class:`dict`): the date the release was made | - **date** (:class:`dict`): the date the release was made | ||||
- **name** (:class:`bytes`): the name of the release | - **name** (:class:`bytes`): the name of the release | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | def release_get_random(self): | ||||
"""Finds a random release id. | """Finds a random release id. | ||||
Returns: | Returns: | ||||
a sha1_git | a sha1_git | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("snapshot/add") | @remote_api_endpoint("snapshot/add") | ||||
def snapshot_add(self, snapshots: Iterable[Snapshot]) -> Dict: | def snapshot_add(self, snapshots: List[Snapshot]) -> Dict: | ||||
"""Add snapshots to the storage. | """Add snapshots to the storage. | ||||
Args: | Args: | ||||
snapshot ([dict]): the snapshots to add, containing the | snapshot ([dict]): the snapshots to add, containing the | ||||
following keys: | following keys: | ||||
- **id** (:class:`bytes`): id of the snapshot | - **id** (:class:`bytes`): id of the snapshot | ||||
- **branches** (:class:`dict`): branches the snapshot contains, | - **branches** (:class:`dict`): branches the snapshot contains, | ||||
▲ Show 20 Lines • Show All 135 Lines • ▼ Show 20 Lines | def snapshot_get_random(self): | ||||
"""Finds a random snapshot id. | """Finds a random snapshot id. | ||||
Returns: | Returns: | ||||
a sha1_git | a sha1_git | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("origin/visit/add") | @remote_api_endpoint("origin/visit/add") | ||||
def origin_visit_add(self, visits: Iterable[OriginVisit]) -> Iterable[OriginVisit]: | def origin_visit_add(self, visits: List[OriginVisit]) -> Iterable[OriginVisit]: | ||||
"""Add visits to storage. If the visits have no id, they will be created and assigned | """Add visits to storage. If the visits have no id, they will be created and assigned | ||||
one. The resulted visits are visits with their visit id set. | one. The resulted visits are visits with their visit id set. | ||||
Args: | Args: | ||||
visits: Iterable of OriginVisit objects to add | visits: List of OriginVisit objects to add | ||||
Raises: | Raises: | ||||
StorageArgumentException if some origin visit reference unknown origins | StorageArgumentException if some origin visit reference unknown origins | ||||
Returns: | Returns: | ||||
Iterable[OriginVisit] stored | List[OriginVisit] stored | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("origin/visit_status/add") | @remote_api_endpoint("origin/visit_status/add") | ||||
def origin_visit_status_add( | def origin_visit_status_add(self, visit_statuses: List[OriginVisitStatus],) -> None: | ||||
self, visit_statuses: Iterable[OriginVisitStatus], | |||||
) -> None: | |||||
"""Add origin visit statuses. | """Add origin visit statuses. | ||||
If there is already a status for the same origin and visit id at the same | If there is already a status for the same origin and visit id at the same | ||||
date, the new one will be either dropped or will replace the existing one | date, the new one will be either dropped or will replace the existing one | ||||
(it is unspecified which one of these two behaviors happens). | (it is unspecified which one of these two behaviors happens). | ||||
Args: | Args: | ||||
visit_statuses: origin visit statuses to add | visit_statuses: origin visit statuses to add | ||||
▲ Show 20 Lines • Show All 146 Lines • ▼ Show 20 Lines | def object_find_by_sha1_git(self, ids): | ||||
- sha1_git: the input id | - sha1_git: the input id | ||||
- type: the type of object found | - type: the type of object found | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("origin/get") | @remote_api_endpoint("origin/get") | ||||
def origin_get(self, origins: Iterable[str]) -> Iterable[Optional[Origin]]: | def origin_get(self, origins: List[str]) -> Iterable[Optional[Origin]]: | ||||
"""Return origins. | """Return origins. | ||||
Args: | Args: | ||||
origin: a list of urls to find | origin: a list of urls to find | ||||
Returns: | Returns: | ||||
the list of associated existing origin model objects. The unknown origins | the list of associated existing origin model objects. The unknown origins | ||||
will be returned as None at the same index as the input. | will be returned as None at the same index as the input. | ||||
▲ Show 20 Lines • Show All 90 Lines • ▼ Show 20 Lines | def origin_count(self, url_pattern, regexp=False, with_visit=False): | ||||
with_visit (bool): if True, filter out origins with no visit | with_visit (bool): if True, filter out origins with no visit | ||||
Returns: | Returns: | ||||
int: The number of origins matching the search criterion. | int: The number of origins matching the search criterion. | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("origin/add_multi") | @remote_api_endpoint("origin/add_multi") | ||||
def origin_add(self, origins: Iterable[Origin]) -> Dict[str, int]: | def origin_add(self, origins: List[Origin]) -> Dict[str, int]: | ||||
"""Add origins to the storage | """Add origins to the storage | ||||
Args: | Args: | ||||
origins: list of dictionaries representing the individual origins, | origins: list of dictionaries representing the individual origins, | ||||
with the following keys: | with the following keys: | ||||
- type: the origin type ('git', 'svn', 'deb', ...) | - type: the origin type ('git', 'svn', 'deb', ...) | ||||
- url (bytes): the url the origin points to | - url (bytes): the url the origin points to | ||||
Show All 16 Lines | def stat_counters(self): | ||||
""" | """ | ||||
... | ... | ||||
def refresh_stat_counters(self): | def refresh_stat_counters(self): | ||||
"""Recomputes the statistics for `stat_counters`.""" | """Recomputes the statistics for `stat_counters`.""" | ||||
... | ... | ||||
@remote_api_endpoint("raw_extrinsic_metadata/add") | @remote_api_endpoint("raw_extrinsic_metadata/add") | ||||
def raw_extrinsic_metadata_add( | def raw_extrinsic_metadata_add(self, metadata: List[RawExtrinsicMetadata],) -> None: | ||||
self, metadata: Iterable[RawExtrinsicMetadata], | |||||
) -> None: | |||||
"""Add extrinsic metadata on objects (contents, directories, ...). | """Add extrinsic metadata on objects (contents, directories, ...). | ||||
The authority and fetcher must be known to the storage before | The authority and fetcher must be known to the storage before | ||||
using this endpoint. | using this endpoint. | ||||
If there is already metadata for the same object, authority, | If there is already metadata for the same object, authority, | ||||
fetcher, and at the same date; the new one will be either dropped or | fetcher, and at the same date; the new one will be either dropped or | ||||
will replace the existing one | will replace the existing one | ||||
Show All 29 Lines | ) -> Dict[str, Union[Optional[bytes], List[RawExtrinsicMetadata]]]: | ||||
`next_page_token` is an opaque token that is used to get the | `next_page_token` is an opaque token that is used to get the | ||||
next page of results, or `None` if there are no more results. | next page of results, or `None` if there are no more results. | ||||
`results` is a list of RawExtrinsicMetadata objects: | `results` is a list of RawExtrinsicMetadata objects: | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("metadata_fetcher/add") | @remote_api_endpoint("metadata_fetcher/add") | ||||
def metadata_fetcher_add(self, fetchers: Iterable[MetadataFetcher],) -> None: | def metadata_fetcher_add(self, fetchers: List[MetadataFetcher],) -> None: | ||||
"""Add new metadata fetchers to the storage. | """Add new metadata fetchers to the storage. | ||||
Their `name` and `version` together are unique identifiers of this | Their `name` and `version` together are unique identifiers of this | ||||
fetcher; and `metadata` is an arbitrary dict of JSONable data | fetcher; and `metadata` is an arbitrary dict of JSONable data | ||||
with information about this fetcher, which must not be `None` | with information about this fetcher, which must not be `None` | ||||
(but may be empty). | (but may be empty). | ||||
Args: | Args: | ||||
Show All 15 Lines | ) -> Optional[MetadataFetcher]: | ||||
Returns: | Returns: | ||||
a MetadataFetcher object (with a non-None metadata field) if it is known, | a MetadataFetcher object (with a non-None metadata field) if it is known, | ||||
else None. | else None. | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("metadata_authority/add") | @remote_api_endpoint("metadata_authority/add") | ||||
def metadata_authority_add(self, authorities: Iterable[MetadataAuthority]) -> None: | def metadata_authority_add(self, authorities: List[MetadataAuthority]) -> None: | ||||
"""Add new metadata authorities to the storage. | """Add new metadata authorities to the storage. | ||||
Their `type` and `url` together are unique identifiers of this | Their `type` and `url` together are unique identifiers of this | ||||
authority; and `metadata` is an arbitrary dict of JSONable data | authority; and `metadata` is an arbitrary dict of JSONable data | ||||
with information about this authority, which must not be `None` | with information about this authority, which must not be `None` | ||||
(but may be empty). | (but may be empty). | ||||
Args: | Args: | ||||
▲ Show 20 Lines • Show All 68 Lines • ▼ Show 20 Lines | def diff_revision(self, revision, track_renaming=False): | ||||
Returns: | Returns: | ||||
A list of dict describing the introduced file changes | A list of dict describing the introduced file changes | ||||
(see :func:`swh.storage.algos.diff.diff_directories` | (see :func:`swh.storage.algos.diff.diff_directories` | ||||
for more details). | for more details). | ||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("clear/buffer") | @remote_api_endpoint("clear/buffer") | ||||
def clear_buffers(self, object_types: Optional[Iterable[str]] = None) -> None: | def clear_buffers(self, object_types: Optional[List[str]] = None) -> None: | ||||
"""For backend storages (pg, storage, in-memory), this is a noop operation. For proxy | """For backend storages (pg, storage, in-memory), this is a noop operation. For proxy | ||||
storages (especially filter, buffer), this is an operation which cleans internal | storages (especially filter, buffer), this is an operation which cleans internal | ||||
state. | state. | ||||
""" | """ | ||||
@remote_api_endpoint("flush") | @remote_api_endpoint("flush") | ||||
def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict: | def flush(self, object_types: Optional[List[str]] = None) -> Dict: | ||||
"""For backend storages (pg, storage, in-memory), this is expected to be a noop | """For backend storages (pg, storage, in-memory), this is expected to be a noop | ||||
operation. For proxy storages (especially buffer), this is expected to trigger | operation. For proxy storages (especially buffer), this is expected to trigger | ||||
actual writes to the backend. | actual writes to the backend. | ||||
""" | """ | ||||
... | ... |