Differential D2988 Diff 10626 swh/storage/interface.py

Changeset View

Standalone View

swh/storage/interface.py

Show First 20 Lines • Show All 1,126 Lines • ▼ Show 20 Lines	def stat_counters(self):
"""		"""
...		...

def refresh_stat_counters(self):		def refresh_stat_counters(self):
"""Recomputes the statistics for `stat_counters`."""		"""Recomputes the statistics for `stat_counters`."""
...		...

@remote_api_endpoint("origin/metadata/add")		@remote_api_endpoint("origin/metadata/add")
def origin_metadata_add(self, origin_url, ts, provider, tool, metadata):		def origin_metadata_add(
""" Add an origin_metadata for the origin at ts with provenance and		self,
metadata.		origin_url: str,
		discovery_date: datetime.datetime,
Args:		authority: Dict[str, Any],
origin_url (str): the origin url for which the metadata is added		fetcher: Dict[str, Any],
ts (datetime): timestamp of the found metadata		format: str,
provider (int): the provider of metadata (ex:'hal')		metadata: bytes,
tool (int): tool used to extract metadata		) -> None:
metadata (jsonb): the metadata retrieved at the time and location		"""Add an origin_metadata for the origin at discovery_date,
		obtained using the `fetcher` from the `authority`.

		The authority and fetcher must be known to the storage before
		using this endpoint.

		Args:
		discovery_date: when the metadata was fetched.
		moraneggUnsubmitted Done Inline Actions and? moranegg: and?
		authority: a dict containing keys `type` and `url`, and
		fetcher: a dict containing keys `name` and `version`.
		format: text field indicating the format of the content of the
		metadata: blob of raw metadata
"""		"""
...		...

@remote_api_endpoint("origin/metadata/get")		@remote_api_endpoint("origin/metadata/get")
def origin_metadata_get_by(self, origin_url, provider_type=None):		def origin_metadata_get(
		self,
		origin_url: str,
		authority: Dict[str, str],
		after: Optional[datetime.datetime] = None,
		limit: Optional[int] = None,
		) -> List[Dict[str, Any]]:
"""Retrieve list of all origin_metadata entries for the origin_id		"""Retrieve list of all origin_metadata entries for the origin_id

Args:		Args:
origin_url (str): the origin's URL		origin_url: the origin's URL
		moraneggUnsubmitted Done Inline Actions is the `and` at the end intentional? moranegg: is the `and` at the end intentional?
		vlorentzAuthorUnsubmitted Done Inline Actions no, that's a typo vlorentz: no, that's a typo
provider_type (str): (optional) type of provider		authority: a dict containing keys `type` and `url`, and
		after: minimum discovery_date for a result to be returned
		limit: maximum number of results to be returned

Returns:		Returns:
list of dicts: the origin_metadata dictionary with the keys:		list of dicts in the format:

		.. code-block: python

- origin_id (int): origin's id		{
- discovery_date (datetime): timestamp of discovery		'authority': {'type': ..., 'url': ...},
- tool_id (int): metadata's extracting tool		'fetcher': {'name': ..., 'version': ...},
- metadata (jsonb)		'discovery_date': ...,
- provider_id (int): metadata's provider		'format': '...',
- provider_name (str)		'metadata': b'...'
- provider_type (str)		}
- provider_url (str)

"""		"""
...		...

@remote_api_endpoint("tool/add")		@remote_api_endpoint("fetcher/add")
def tool_add(self, tools):		def metadata_fetcher_add(
"""Add new tools to the storage.		self, name: str, version: str, metadata: Dict[str, Any]
		) -> None:
		"""Add a new metadata fetcher to the storage.

Args:		`name` and `version` together are a unique identifier of this
tools (iterable of :class:`dict`): Tool information to add to		fetcher; and `metadata` is an arbitrary dict of JSONable data
storage. Each tool is a :class:`dict` with the following keys:		with information about this fetcher.

- name (:class:`str`): name of the tool
- version (:class:`str`): version of the tool
- configuration (:class:`dict`): configuration of the tool,
must be json-encodable

Returns:		Args:
:class:`dict`: All the tools inserted in storage		name: the name of the fetcher
(including the internal ``id``). The order of the list is not		version: version of the fetcher
guaranteed to match the order of the initial list.

"""		"""
...		...

@remote_api_endpoint("tool/data")		@remote_api_endpoint("fetcher/get")
def tool_get(self, tool):		def metadata_fetcher_get(self, name: str, version: str) -> Optional[Dict[str, Any]]:
"""Retrieve tool information.		"""Retrieve information about a fetcher

Args:		Args:
tool (dict): Tool information we want to retrieve from storage.		name: the name of the fetcher
The dicts have the same keys as those used in :func:`tool_add`.		version: version of the fetcher

Returns:		Returns:
dict: The full tool information if it exists (``id`` included),		dictionary with keys `name`, `version`, and `metadata`; or None
None otherwise.		if the fetcher is not known

"""		"""
...		...

@remote_api_endpoint("provider/add")		@remote_api_endpoint("authority/add")
def metadata_provider_add(		def metadata_authority_add(
self, provider_name, provider_type, provider_url, metadata		self, type: str, url: str, metadata: Dict[str, Any]
):		) -> None:
"""Add a metadata provider.		"""Add a metadata authority

Args:		Args:
provider_name (str): Its name		type: one of "deposit", "forge", or "registry"
provider_type (str): Its type (eg. `'deposit-client'`)		url: unique URI identifying the authority
provider_url (str): Its URL
metadata: JSON-encodable object		metadata: JSON-encodable object

Returns:
int: an identifier of the provider
"""
...

@remote_api_endpoint("provider/get")
def metadata_provider_get(self, provider_id):
"""Get a metadata provider

Args:
provider_id: Its identifier, as given by `metadata_provider_add`.

Returns:
dict: same as `metadata_provider_add`;
or None if it does not exist.
"""		"""
...		...

@remote_api_endpoint("provider/getby")		@remote_api_endpoint("authority/get")
def metadata_provider_get_by(self, provider):		def metadata_authority_get(self, type: str, url: str) -> Optional[Dict[str, Any]]:
"""Get a metadata provider		"""Retrieve information about an authority

Args:		Args:
provider (dict): A dictionary with keys:		type: one of "deposit", "forge", or "registry"
* provider_name: Its name		url: unique URI identifying the authority
		moraneggUnsubmitted Not Done Inline Actions is it a fixed set? moranegg: is it a fixed set?
		vlorentzAuthorUnsubmitted Done Inline Actions According to the spec, yes. We can extend it later anyway. vlorentz: According to the spec, yes. We can extend it later anyway.
* provider_url: Its URL

Returns:		Returns:
dict: same as `metadata_provider_add`;		dictionary with keys `type`, `url`, and `metadata`; or None
or None if it does not exist.		if the authority is not known
"""		"""
...		...

@deprecated		@deprecated
@remote_api_endpoint("algos/diff_directories")		@remote_api_endpoint("algos/diff_directories")
def diff_directories(self, from_dir, to_dir, track_renaming=False):		def diff_directories(self, from_dir, to_dir, track_renaming=False):
"""Compute the list of file changes introduced between two arbitrary		"""Compute the list of file changes introduced between two arbitrary
directories (insertion / deletion / modification / renaming of files).		directories (insertion / deletion / modification / renaming of files).
▲ Show 20 Lines • Show All 58 Lines • Show Last 20 Lines