Differential D4158 Diff 14770 swh/indexer/storage/interface.py

Changeset View

Standalone View

swh/indexer/storage/interface.py

# Copyright (C) 2015-2020 The Software Heritage developers		# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from typing import Dict, List, Optional, TypeVar		from typing import Dict, Iterable, Iterator, List, Optional, Tuple, TypeVar

from swh.core.api import remote_api_endpoint		from swh.core.api import remote_api_endpoint
from swh.core.api.classes import PagedResult as CorePagedResult		from swh.core.api.classes import PagedResult as CorePagedResult
		from swh.indexer.storage.model import ContentMimetypeRow

TResult = TypeVar("TResult")		TResult = TypeVar("TResult")
PagedResult = CorePagedResult[TResult, str]		PagedResult = CorePagedResult[TResult, str]


Sha1 = bytes		Sha1 = bytes


class IndexerStorageInterface:		class IndexerStorageInterface:
@remote_api_endpoint("check_config")		@remote_api_endpoint("check_config")
def check_config(self, *, check_write):		def check_config(self, *, check_write):
"""Check that the storage is configured and ready to go."""		"""Check that the storage is configured and ready to go."""
...		...

@remote_api_endpoint("content_mimetype/missing")		@remote_api_endpoint("content_mimetype/missing")
def content_mimetype_missing(self, mimetypes):		def content_mimetype_missing(
		self, mimetypes: Iterable[Dict]
		) -> Iterator[Tuple[Sha1, int]]:
"""Generate mimetypes missing from storage.		"""Generate mimetypes missing from storage.

Args:		Args:
mimetypes (iterable): iterable of dict with keys:		mimetypes (iterable): iterable of dict with keys:

- id (bytes): sha1 identifier		- id (bytes): sha1 identifier
- indexer_configuration_id (int): tool used to compute the		- indexer_configuration_id (int): tool used to compute the
results		results
Show All 31 Lines	) -> PagedResult[Sha1]:
PagedResult of Sha1. If next_page_token is None, there is no more data		PagedResult of Sha1. If next_page_token is None, there is no more data
to fetch		to fetch

"""		"""
...		...

@remote_api_endpoint("content_mimetype/add")		@remote_api_endpoint("content_mimetype/add")
def content_mimetype_add(		def content_mimetype_add(
self, mimetypes: List[Dict], conflict_update: bool = False		self, mimetypes: List[ContentMimetypeRow], conflict_update: bool = False
) -> Dict[str, int]:		) -> Dict[str, int]:
"""Add mimetypes not present in storage.		"""Add mimetypes not present in storage.

Args:		Args:
mimetypes (iterable): dictionaries with keys:		mimetypes: mimetype rows to be added
		conflict_update: Flag to determine if we want to
- id (bytes): sha1 identifier
- mimetype (bytes): raw content's mimetype
- encoding (bytes): raw content's encoding
- indexer_configuration_id (int): tool's id used to
compute the results
- conflict_update (bool): Flag to determine if we want to
overwrite (``True``) or skip duplicates (``False``, the		overwrite (``True``) or skip duplicates (``False``, the
default)		default)

Returns:		Returns:
Dict summary of number of rows added		Dict summary of number of rows added

"""		"""
...		...

@remote_api_endpoint("content_mimetype")		@remote_api_endpoint("content_mimetype")
def content_mimetype_get(self, ids):		def content_mimetype_get(self, ids: Iterable[Sha1]) -> Iterator[ContentMimetypeRow]:
"""Retrieve full content mimetype per ids.		"""Retrieve full content mimetype per ids.

Args:		Args:
ids (iterable): sha1 identifier		ids (iterable): sha1 identifier

Yields:		Yields:
mimetypes (iterable): dictionaries with keys:		mimetypes (iterable): dictionaries with keys:

▲ Show 20 Lines • Show All 512 Lines • Show Last 20 Lines