Differential D3247 Diff 11900 swh/storage/interface.py

Changeset View

Standalone View

swh/storage/interface.py

# Copyright (C) 2015-2020 The Software Heritage developers		# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import datetime		import datetime

from typing import Any, Dict, Iterable, List, Optional		from typing import Any, Dict, Iterable, List, Optional, Union

from swh.core.api import remote_api_endpoint		from swh.core.api import remote_api_endpoint
from swh.model.model import (		from swh.model.model import (
Content,		Content,
Directory,		Directory,
Origin,		Origin,
OriginVisit,		OriginVisit,
OriginVisitStatus,		OriginVisitStatus,
▲ Show 20 Lines • Show All 1,076 Lines • ▼ Show 20 Lines	def stat_counters(self):

"""		"""
...		...

def refresh_stat_counters(self):		def refresh_stat_counters(self):
"""Recomputes the statistics for `stat_counters`."""		"""Recomputes the statistics for `stat_counters`."""
...		...

		@remote_api_endpoint("content/metadata/add")
		def content_metadata_add(
		self,
		id: str,
		context: Dict[str, Union[str, bytes, int]],
		discovery_date: datetime.datetime,
		authority: Dict[str, Any],
		fetcher: Dict[str, Any],
		format: str,
		metadata: bytes,
		) -> None:
		"""Add a content_metadata for the content at discovery_date,
		obtained using the `fetcher` from the `authority`.

		The authority and fetcher must be known to the storage before
		using this endpoint.

		If there is already content metadata for the same content, authority,
		fetcher, and at the same date, it will be replaced by this one.

		Args:
		discovery_date: when the metadata was fetched.
		authority: a dict containing keys `type` and `url`.
		fetcher: a dict containing keys `name` and `version`.
		format: text field indicating the format of the content of the
		metadata: blob of raw metadata
		"""
		...

		@remote_api_endpoint("content/metadata/get")
		def content_metadata_get(
		self,
		id: str,
		authority: Dict[str, str],
		after: Optional[datetime.datetime] = None,
		page_token: Optional[bytes] = None,
		limit: int = 1000,
		) -> Dict[str, Any]:
		"""Retrieve list of all content_metadata entries for the id

		Args:
		id: the content's SWHID
		authority: a dict containing keys `type` and `url`.
		after: minimum discovery_date for a result to be returned
		page_token: opaque token, used to get the next page of results
		limit: maximum number of results to be returned

		Returns:
		dict with keys `next_page_token` and `results`.
		`next_page_token` is an opaque token that is used to get the
		next page of results, or `None` if there are no more results.
		`results` is a list of dicts in the format:

		.. code-block: python

		{
		'authority': {'type': ..., 'url': ...},
		'fetcher': {'name': ..., 'version': ...},
		'discovery_date': ...,
		'format': '...',
		'metadata': b'...',
		'context': { ... },
		}

		"""
		...

@remote_api_endpoint("origin/metadata/add")		@remote_api_endpoint("origin/metadata/add")
def origin_metadata_add(		def origin_metadata_add(
self,		self,
origin_url: str,		origin_url: str,
discovery_date: datetime.datetime,		discovery_date: datetime.datetime,
authority: Dict[str, Any],		authority: Dict[str, Any],
fetcher: Dict[str, Any],		fetcher: Dict[str, Any],
format: str,		format: str,
Show All 21 Lines
def origin_metadata_get(		def origin_metadata_get(
self,		self,
origin_url: str,		origin_url: str,
authority: Dict[str, str],		authority: Dict[str, str],
after: Optional[datetime.datetime] = None,		after: Optional[datetime.datetime] = None,
page_token: Optional[bytes] = None,		page_token: Optional[bytes] = None,
limit: int = 1000,		limit: int = 1000,
) -> Dict[str, Any]:		) -> Dict[str, Any]:
"""Retrieve list of all origin_metadata entries for the origin_id		"""Retrieve list of all origin_metadata entries for the origin_url

Args:		Args:
origin_url: the origin's URL		origin_url: the origin's URL
authority: a dict containing keys `type` and `url`.		authority: a dict containing keys `type` and `url`.
after: minimum discovery_date for a result to be returned		after: minimum discovery_date for a result to be returned
page_token: opaque token, used to get the next page of results		page_token: opaque token, used to get the next page of results
limit: maximum number of results to be returned		limit: maximum number of results to be returned

▲ Show 20 Lines • Show All 148 Lines • Show Last 20 Lines