Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/in_memory.py
| Show First 20 Lines • Show All 96 Lines • ▼ Show 20 Lines | def add(self, data, conflict_update): | ||||
| self._tools_per_id[id_].add(tool_id) | self._tools_per_id[id_].add(tool_id) | ||||
| class IndexerStorage: | class IndexerStorage: | ||||
| """In-memory SWH indexer storage.""" | """In-memory SWH indexer storage.""" | ||||
| def __init__(self): | def __init__(self): | ||||
| self._tools = {} | self._tools = {} | ||||
| self._mimetypes = SubStorage(self._tools) | |||||
| self._content_ctags = SubStorage(self._tools) | self._content_ctags = SubStorage(self._tools) | ||||
| self._content_metadata = SubStorage(self._tools) | self._content_metadata = SubStorage(self._tools) | ||||
| self._revision_metadata = SubStorage(self._tools) | self._revision_metadata = SubStorage(self._tools) | ||||
| def content_mimetype_missing(self, mimetypes): | |||||
| """Generate mimetypes missing from storage. | |||||
| Args: | |||||
| mimetypes (iterable): iterable of dict with keys: | |||||
| - **id** (bytes): sha1 identifier | |||||
| - **indexer_configuration_id** (int): tool used to compute the | |||||
| results | |||||
| Yields: | |||||
| tuple (id, indexer_configuration_id): missing id | |||||
| """ | |||||
| yield from self._mimetypes.missing(mimetypes) | |||||
| def content_mimetype_add(self, mimetypes, conflict_update=False): | |||||
| """Add mimetypes not present in storage. | |||||
| Args: | |||||
| mimetypes (iterable): dictionaries with keys: | |||||
| - **id** (bytes): sha1 identifier | |||||
| - **mimetype** (bytes): raw content's mimetype | |||||
| - **encoding** (bytes): raw content's encoding | |||||
| - **indexer_configuration_id** (int): tool's id used to | |||||
| compute the results | |||||
| - **conflict_update** (bool): Flag to determine if we want to | |||||
| overwrite (``True``) or skip duplicates (``False``, the | |||||
| default) | |||||
| """ | |||||
| self._mimetypes.add(mimetypes, conflict_update) | |||||
| def content_mimetype_get(self, ids, db=None, cur=None): | |||||
| """Retrieve full content mimetype per ids. | |||||
| Args: | |||||
| ids (iterable): sha1 identifier | |||||
| Yields: | |||||
| mimetypes (iterable): dictionaries with keys: | |||||
| - **id** (bytes): sha1 identifier | |||||
| - **mimetype** (bytes): raw content's mimetype | |||||
| - **encoding** (bytes): raw content's encoding | |||||
| - **tool** (dict): Tool used to compute the language | |||||
| """ | |||||
| yield from self._mimetypes.get(ids) | |||||
| def content_ctags_missing(self, ctags): | def content_ctags_missing(self, ctags): | ||||
| """List ctags missing from storage. | """List ctags missing from storage. | ||||
| Args: | Args: | ||||
| ctags (iterable): dicts with keys: | ctags (iterable): dicts with keys: | ||||
| - **id** (bytes): sha1 identifier | - **id** (bytes): sha1 identifier | ||||
| - **indexer_configuration_id** (int): tool used to compute | - **indexer_configuration_id** (int): tool used to compute | ||||
| ▲ Show 20 Lines • Show All 249 Lines • Show Last 20 Lines | |||||