Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/in_memory.py
Show First 20 Lines • Show All 606 Lines • ▼ Show 20 Lines | def origin_intrinsic_metadata_get(self, ids): | ||||
Args: | Args: | ||||
ids (iterable): origin identifiers | ids (iterable): origin identifiers | ||||
Yields: | Yields: | ||||
list: dictionaries with the following keys: | list: dictionaries with the following keys: | ||||
- **id** (int) | - **id** (int) | ||||
- **origin_url** (str) | |||||
- **from_revision** (bytes): which revision this metadata | |||||
was extracted from | |||||
- **metadata** (str): associated metadata | - **metadata** (str): associated metadata | ||||
- **tool** (dict): tool used to compute metadata | - **tool** (dict): tool used to compute metadata | ||||
- **mappings** (List[str]): list of mappings used to translate | - **mappings** (List[str]): list of mappings used to translate | ||||
these metadata | these metadata | ||||
""" | """ | ||||
yield from self._origin_intrinsic_metadata.get(ids) | yield from self._origin_intrinsic_metadata.get(ids) | ||||
def origin_intrinsic_metadata_add(self, metadata, | def origin_intrinsic_metadata_add(self, metadata, | ||||
conflict_update=False): | conflict_update=False): | ||||
"""Add origin metadata not present in storage. | """Add origin metadata not present in storage. | ||||
Args: | Args: | ||||
metadata (iterable): dictionaries with keys: | metadata (iterable): dictionaries with keys: | ||||
- **id**: origin identifier | - **id**: origin identifier | ||||
- **origin_url** | |||||
- **from_revision**: sha1 id of the revision used to generate | - **from_revision**: sha1 id of the revision used to generate | ||||
these metadata. | these metadata. | ||||
- **metadata**: arbitrary dict | - **metadata**: arbitrary dict | ||||
- **indexer_configuration_id**: tool used to compute metadata | - **indexer_configuration_id**: tool used to compute metadata | ||||
- **mappings** (List[str]): list of mappings used to translate | - **mappings** (List[str]): list of mappings used to translate | ||||
these metadata | these metadata | ||||
conflict_update: Flag to determine if we want to overwrite (true) | conflict_update: Flag to determine if we want to overwrite (true) | ||||
Show All 20 Lines | def origin_intrinsic_metadata_search_fulltext( | ||||
Args: | Args: | ||||
conjunction (List[str]): List of terms to be searched for. | conjunction (List[str]): List of terms to be searched for. | ||||
limit (int): The maximum number of results to return | limit (int): The maximum number of results to return | ||||
Yields: | Yields: | ||||
list: dictionaries with the following keys: | list: dictionaries with the following keys: | ||||
- **id** (int) | - **id** (int) | ||||
- **origin_url** (str) | |||||
- **from_revision** (bytes): which revision this metadata | |||||
was extracted from | |||||
- **metadata** (str): associated metadata | - **metadata** (str): associated metadata | ||||
- **tool** (dict): tool used to compute metadata | - **tool** (dict): tool used to compute metadata | ||||
- **mappings** (List[str]): list of mappings used to translate | - **mappings** (List[str]): list of mappings used to translate | ||||
these metadata | these metadata | ||||
""" | """ | ||||
# A very crude fulltext search implementation, but that's enough | # A very crude fulltext search implementation, but that's enough | ||||
# to work on English metadata | # to work on English metadata | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | def origin_intrinsic_metadata_search_by_producer( | ||||
mappings (List[str]): Returns origins whose intrinsic metadata | mappings (List[str]): Returns origins whose intrinsic metadata | ||||
were generated using at least one of these mappings. | were generated using at least one of these mappings. | ||||
Yields: | Yields: | ||||
list: list of origin ids (int) if `ids_only=True`, else | list: list of origin ids (int) if `ids_only=True`, else | ||||
dictionaries with the following keys: | dictionaries with the following keys: | ||||
- **id** (int) | - **id** (int) | ||||
- **origin_url** (str) | |||||
- **from_revision**: sha1 id of the revision used to generate | |||||
these metadata. | |||||
- **metadata** (str): associated metadata | - **metadata** (str): associated metadata | ||||
- **tool** (dict): tool used to compute metadata | - **tool** (dict): tool used to compute metadata | ||||
- **mappings** (List[str]): list of mappings used to translate | - **mappings** (List[str]): list of mappings used to translate | ||||
these metadata | these metadata | ||||
""" | """ | ||||
nb_results = 0 | nb_results = 0 | ||||
if mappings is not None: | if mappings is not None: | ||||
▲ Show 20 Lines • Show All 95 Lines • Show Last 20 Lines |