Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/interface.py
# Copyright (C) 2015-2020 The Software Heritage developers | # Copyright (C) 2015-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import datetime | import datetime | ||||
from typing import Any, Dict, Iterable, List, Optional | from typing import Any, Dict, Iterable, List, Optional, Union | ||||
from swh.core.api import remote_api_endpoint | from swh.core.api import remote_api_endpoint | ||||
from swh.model.model import ( | from swh.model.model import ( | ||||
Content, | Content, | ||||
Directory, | Directory, | ||||
Origin, | Origin, | ||||
OriginVisit, | OriginVisit, | ||||
OriginVisitStatus, | OriginVisitStatus, | ||||
▲ Show 20 Lines • Show All 1,076 Lines • ▼ Show 20 Lines | def stat_counters(self): | ||||
""" | """ | ||||
... | ... | ||||
def refresh_stat_counters(self): | def refresh_stat_counters(self): | ||||
"""Recomputes the statistics for `stat_counters`.""" | """Recomputes the statistics for `stat_counters`.""" | ||||
... | ... | ||||
@remote_api_endpoint("content/metadata/add") | |||||
def content_metadata_add( | |||||
self, | |||||
id: str, | |||||
context: Dict[str, Union[str, bytes, int]], | |||||
discovery_date: datetime.datetime, | |||||
authority: Dict[str, Any], | |||||
fetcher: Dict[str, Any], | |||||
format: str, | |||||
metadata: bytes, | |||||
) -> None: | |||||
"""Add a content_metadata for the content at discovery_date, | |||||
obtained using the `fetcher` from the `authority`. | |||||
The authority and fetcher must be known to the storage before | |||||
using this endpoint. | |||||
If there is already content metadata for the same content, authority, | |||||
fetcher, and at the same date, it will be replaced by this one. | |||||
Args: | |||||
discovery_date: when the metadata was fetched. | |||||
authority: a dict containing keys `type` and `url`. | |||||
fetcher: a dict containing keys `name` and `version`. | |||||
format: text field indicating the format of the content of the | |||||
metadata: blob of raw metadata | |||||
""" | |||||
... | |||||
@remote_api_endpoint("content/metadata/get") | |||||
def content_metadata_get( | |||||
self, | |||||
id: str, | |||||
authority: Dict[str, str], | |||||
after: Optional[datetime.datetime] = None, | |||||
page_token: Optional[bytes] = None, | |||||
limit: int = 1000, | |||||
) -> Dict[str, Any]: | |||||
"""Retrieve list of all content_metadata entries for the id | |||||
Args: | |||||
id: the content's SWHID | |||||
authority: a dict containing keys `type` and `url`. | |||||
after: minimum discovery_date for a result to be returned | |||||
page_token: opaque token, used to get the next page of results | |||||
limit: maximum number of results to be returned | |||||
Returns: | |||||
dict with keys `next_page_token` and `results`. | |||||
`next_page_token` is an opaque token that is used to get the | |||||
next page of results, or `None` if there are no more results. | |||||
`results` is a list of dicts in the format: | |||||
.. code-block: python | |||||
{ | |||||
'authority': {'type': ..., 'url': ...}, | |||||
'fetcher': {'name': ..., 'version': ...}, | |||||
'discovery_date': ..., | |||||
'format': '...', | |||||
'metadata': b'...', | |||||
'context': { ... }, | |||||
} | |||||
""" | |||||
... | |||||
@remote_api_endpoint("origin/metadata/add") | @remote_api_endpoint("origin/metadata/add") | ||||
def origin_metadata_add( | def origin_metadata_add( | ||||
self, | self, | ||||
origin_url: str, | origin_url: str, | ||||
discovery_date: datetime.datetime, | discovery_date: datetime.datetime, | ||||
authority: Dict[str, Any], | authority: Dict[str, Any], | ||||
fetcher: Dict[str, Any], | fetcher: Dict[str, Any], | ||||
format: str, | format: str, | ||||
▲ Show 20 Lines • Show All 186 Lines • Show Last 20 Lines |