Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/__init__.py
Show First 20 Lines • Show All 707 Lines • ▼ Show 20 Lines | class IndexerStorage: | ||||
def origin_intrinsic_metadata_add(self, metadata, | def origin_intrinsic_metadata_add(self, metadata, | ||||
conflict_update=False, db=None, | conflict_update=False, db=None, | ||||
cur=None): | cur=None): | ||||
"""Add origin metadata not present in storage. | """Add origin metadata not present in storage. | ||||
Args: | Args: | ||||
metadata (iterable): dictionaries with keys: | metadata (iterable): dictionaries with keys: | ||||
- **id**: origin identifier | - **id**: legacy origin identifier | ||||
- **origin_url**: URL of the origin | |||||
- **from_revision**: sha1 id of the revision used to generate | - **from_revision**: sha1 id of the revision used to generate | ||||
these metadata. | these metadata. | ||||
- **metadata**: arbitrary dict | - **metadata**: arbitrary dict | ||||
- **indexer_configuration_id**: tool used to compute metadata | - **indexer_configuration_id**: tool used to compute metadata | ||||
- **mappings** (List[str]): list of mappings used to translate | - **mappings** (List[str]): list of mappings used to translate | ||||
these metadata | these metadata | ||||
conflict_update: Flag to determine if we want to overwrite (true) | conflict_update: Flag to determine if we want to overwrite (true) | ||||
or skip duplicates (false, the default) | or skip duplicates (false, the default) | ||||
""" | """ | ||||
_check_id_duplicates(metadata) | _check_id_duplicates(metadata) | ||||
metadata.sort(key=lambda m: m['id']) | metadata.sort(key=lambda m: m['id']) | ||||
db.mktemp_origin_intrinsic_metadata(cur) | db.mktemp_origin_intrinsic_metadata(cur) | ||||
db.copy_to(metadata, 'tmp_origin_intrinsic_metadata', | db.copy_to(metadata, 'tmp_origin_intrinsic_metadata', | ||||
['id', 'metadata', 'indexer_configuration_id', | ['id', 'origin_url', 'metadata', | ||||
'indexer_configuration_id', | |||||
'from_revision', 'mappings'], | 'from_revision', 'mappings'], | ||||
cur) | cur) | ||||
db.origin_intrinsic_metadata_add_from_temp(conflict_update, cur) | db.origin_intrinsic_metadata_add_from_temp(conflict_update, cur) | ||||
@remote_api_endpoint('origin_intrinsic_metadata/delete') | @remote_api_endpoint('origin_intrinsic_metadata/delete') | ||||
@db_transaction() | @db_transaction() | ||||
def origin_intrinsic_metadata_delete( | def origin_intrinsic_metadata_delete( | ||||
self, entries, db=None, cur=None): | self, entries, db=None, cur=None): | ||||
Show All 15 Lines | def origin_intrinsic_metadata_search_fulltext( | ||||
Args: | Args: | ||||
conjunction (List[str]): List of terms to be searched for. | conjunction (List[str]): List of terms to be searched for. | ||||
limit (int): The maximum number of results to return | limit (int): The maximum number of results to return | ||||
Yields: | Yields: | ||||
list: dictionaries with the following keys: | list: dictionaries with the following keys: | ||||
- **id** (int) | - **id** (int): legacy origin identifier | ||||
ardumont: Unify with the other docstring one way (append `: Url of the origin`) or the other (drop the… | |||||
- **origin_url** (str) | |||||
- **metadata** (str): associated metadata | - **metadata** (str): associated metadata | ||||
- **tool** (dict): tool used to compute metadata | - **tool** (dict): tool used to compute metadata | ||||
- **mappings** (List[str]): list of mappings used to translate | - **mappings** (List[str]): list of mappings used to translate | ||||
these metadata | these metadata | ||||
""" | """ | ||||
for c in db.origin_intrinsic_metadata_search_fulltext( | for c in db.origin_intrinsic_metadata_search_fulltext( | ||||
conjunction, limit=limit, cur=cur): | conjunction, limit=limit, cur=cur): | ||||
Show All 16 Lines | def origin_intrinsic_metadata_search_by_producer( | ||||
or the content as well | or the content as well | ||||
mappings (List[str]): Returns origins whose intrinsic metadata | mappings (List[str]): Returns origins whose intrinsic metadata | ||||
were generated using at least one of these mappings. | were generated using at least one of these mappings. | ||||
Yields: | Yields: | ||||
list: list of origin ids (int) if `ids_only=True`, else | list: list of origin ids (int) if `ids_only=True`, else | ||||
dictionaries with the following keys: | dictionaries with the following keys: | ||||
- **id** (int) | - **id** (int): legacy origin identifier | ||||
- **origin_url** (str) | |||||
- **metadata** (str): associated metadata | - **metadata** (str): associated metadata | ||||
- **tool** (dict): tool used to compute metadata | - **tool** (dict): tool used to compute metadata | ||||
- **mappings** (List[str]): list of mappings used to translate | - **mappings** (List[str]): list of mappings used to translate | ||||
these metadata | these metadata | ||||
""" | """ | ||||
res = db.origin_intrinsic_metadata_search_by_producer( | res = db.origin_intrinsic_metadata_search_by_producer( | ||||
start, end, limit, ids_only, mappings, tool_ids, cur) | start, end, limit, ids_only, mappings, tool_ids, cur) | ||||
▲ Show 20 Lines • Show All 110 Lines • Show Last 20 Lines |
Unify with the other docstring one way (append : Url of the origin) or the other (drop the redundant definition)...
/me singing You've got the power! tududu du du du tududu du du du ;)
(~> lookup snap music if you don't grok that ;)