Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/in_memory.py
Show First 20 Lines • Show All 96 Lines • ▼ Show 20 Lines | def add(self, data, conflict_update): | ||||
self._tools_per_id[id_].add(tool_id) | self._tools_per_id[id_].add(tool_id) | ||||
class IndexerStorage: | class IndexerStorage: | ||||
"""In-memory SWH indexer storage.""" | """In-memory SWH indexer storage.""" | ||||
def __init__(self): | def __init__(self): | ||||
self._tools = {} | self._tools = {} | ||||
self._mimetypes = SubStorage(self._tools) | |||||
self._content_ctags = SubStorage(self._tools) | self._content_ctags = SubStorage(self._tools) | ||||
self._content_metadata = SubStorage(self._tools) | self._content_metadata = SubStorage(self._tools) | ||||
self._revision_metadata = SubStorage(self._tools) | self._revision_metadata = SubStorage(self._tools) | ||||
def content_mimetype_missing(self, mimetypes): | |||||
"""Generate mimetypes missing from storage. | |||||
Args: | |||||
mimetypes (iterable): iterable of dict with keys: | |||||
- **id** (bytes): sha1 identifier | |||||
- **indexer_configuration_id** (int): tool used to compute the | |||||
results | |||||
Yields: | |||||
tuple (id, indexer_configuration_id): missing id | |||||
""" | |||||
yield from self._mimetypes.missing(mimetypes) | |||||
def content_mimetype_add(self, mimetypes, conflict_update=False): | |||||
"""Add mimetypes not present in storage. | |||||
Args: | |||||
mimetypes (iterable): dictionaries with keys: | |||||
- **id** (bytes): sha1 identifier | |||||
- **mimetype** (bytes): raw content's mimetype | |||||
- **encoding** (bytes): raw content's encoding | |||||
- **indexer_configuration_id** (int): tool's id used to | |||||
compute the results | |||||
- **conflict_update** (bool): Flag to determine if we want to | |||||
overwrite (``True``) or skip duplicates (``False``, the | |||||
default) | |||||
""" | |||||
self._mimetypes.add(mimetypes, conflict_update) | |||||
def content_mimetype_get(self, ids, db=None, cur=None): | |||||
"""Retrieve full content mimetype per ids. | |||||
Args: | |||||
ids (iterable): sha1 identifier | |||||
Yields: | |||||
mimetypes (iterable): dictionaries with keys: | |||||
- **id** (bytes): sha1 identifier | |||||
- **mimetype** (bytes): raw content's mimetype | |||||
- **encoding** (bytes): raw content's encoding | |||||
- **tool** (dict): Tool used to compute the language | |||||
""" | |||||
yield from self._mimetypes.get(ids) | |||||
def content_ctags_missing(self, ctags): | def content_ctags_missing(self, ctags): | ||||
"""List ctags missing from storage. | """List ctags missing from storage. | ||||
Args: | Args: | ||||
ctags (iterable): dicts with keys: | ctags (iterable): dicts with keys: | ||||
- **id** (bytes): sha1 identifier | - **id** (bytes): sha1 identifier | ||||
- **indexer_configuration_id** (int): tool used to compute | - **indexer_configuration_id** (int): tool used to compute | ||||
▲ Show 20 Lines • Show All 249 Lines • Show Last 20 Lines |