Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/__init__.py
| Show First 20 Lines • Show All 580 Lines • ▼ Show 20 Lines | def revision_metadata_get(self, ids, db=None, cur=None): | ||||
| ids (iterable): sha1 checksums | ids (iterable): sha1 checksums | ||||
| Yields: | Yields: | ||||
| dictionaries with the following keys: | dictionaries with the following keys: | ||||
| - **id** (bytes) | - **id** (bytes) | ||||
| - **translated_metadata** (str): associated metadata | - **translated_metadata** (str): associated metadata | ||||
| - **tool** (dict): tool used to compute metadata | - **tool** (dict): tool used to compute metadata | ||||
| - **mappings** (List[str]): list of mappings used to translate | |||||
| these metadata | |||||
| """ | """ | ||||
| for c in db.revision_metadata_get_from_list(ids, cur): | for c in db.revision_metadata_get_from_list(ids, cur): | ||||
| yield converters.db_to_metadata( | yield converters.db_to_metadata( | ||||
| dict(zip(db.revision_metadata_cols, c))) | dict(zip(db.revision_metadata_cols, c))) | ||||
| @remote_api_endpoint('revision_metadata/add') | @remote_api_endpoint('revision_metadata/add') | ||||
| @db_transaction() | @db_transaction() | ||||
| def revision_metadata_add(self, metadata, conflict_update=False, db=None, | def revision_metadata_add(self, metadata, conflict_update=False, db=None, | ||||
| cur=None): | cur=None): | ||||
| """Add metadata not present in storage. | """Add metadata not present in storage. | ||||
| Args: | Args: | ||||
| metadata (iterable): dictionaries with keys: | metadata (iterable): dictionaries with keys: | ||||
| - **id**: sha1_git of revision | - **id**: sha1_git of revision | ||||
| - **translated_metadata**: arbitrary dict | - **translated_metadata**: arbitrary dict | ||||
| - **indexer_configuration_id**: tool used to compute metadata | - **indexer_configuration_id**: tool used to compute metadata | ||||
| - **mappings** (List[str]): list of mappings used to translate | |||||
| these metadata | |||||
| conflict_update: Flag to determine if we want to overwrite (true) | conflict_update: Flag to determine if we want to overwrite (true) | ||||
| or skip duplicates (false, the default) | or skip duplicates (false, the default) | ||||
| """ | """ | ||||
| db.mktemp_revision_metadata(cur) | db.mktemp_revision_metadata(cur) | ||||
| db.copy_to(metadata, 'tmp_revision_metadata', | db.copy_to(metadata, 'tmp_revision_metadata', | ||||
ardumont: please remove those print statements. | |||||
| ['id', 'translated_metadata', 'indexer_configuration_id'], | ['id', 'translated_metadata', 'mappings', | ||||
| 'indexer_configuration_id'], | |||||
| cur) | cur) | ||||
| db.revision_metadata_add_from_temp(conflict_update, cur) | db.revision_metadata_add_from_temp(conflict_update, cur) | ||||
| @remote_api_endpoint('origin_intrinsic_metadata') | @remote_api_endpoint('origin_intrinsic_metadata') | ||||
| @db_transaction_generator() | @db_transaction_generator() | ||||
| def origin_intrinsic_metadata_get(self, ids, db=None, cur=None): | def origin_intrinsic_metadata_get(self, ids, db=None, cur=None): | ||||
| """Retrieve origin metadata per id. | """Retrieve origin metadata per id. | ||||
| Args: | Args: | ||||
| ids (iterable): origin identifiers | ids (iterable): origin identifiers | ||||
| Yields: | Yields: | ||||
| list: dictionaries with the following keys: | list: dictionaries with the following keys: | ||||
| - **origin_id** (int) | - **origin_id** (int) | ||||
| - **metadata** (str): associated metadata | - **metadata** (str): associated metadata | ||||
| - **tool** (dict): tool used to compute metadata | - **tool** (dict): tool used to compute metadata | ||||
| - **mappings** (List[str]): list of mappings used to translate | |||||
| these metadata | |||||
| """ | """ | ||||
| for c in db.origin_intrinsic_metadata_get_from_list(ids, cur): | for c in db.origin_intrinsic_metadata_get_from_list(ids, cur): | ||||
| yield converters.db_to_metadata( | yield converters.db_to_metadata( | ||||
| dict(zip(db.origin_intrinsic_metadata_cols, c))) | dict(zip(db.origin_intrinsic_metadata_cols, c))) | ||||
| @remote_api_endpoint('origin_intrinsic_metadata/add') | @remote_api_endpoint('origin_intrinsic_metadata/add') | ||||
| @db_transaction() | @db_transaction() | ||||
| def origin_intrinsic_metadata_add(self, metadata, | def origin_intrinsic_metadata_add(self, metadata, | ||||
| conflict_update=False, db=None, | conflict_update=False, db=None, | ||||
| cur=None): | cur=None): | ||||
| """Add origin metadata not present in storage. | """Add origin metadata not present in storage. | ||||
| Args: | Args: | ||||
| metadata (iterable): dictionaries with keys: | metadata (iterable): dictionaries with keys: | ||||
| - **origin_id**: origin identifier | - **origin_id**: origin identifier | ||||
| - **from_revision**: sha1 id of the revision used to generate | - **from_revision**: sha1 id of the revision used to generate | ||||
| these metadata. | these metadata. | ||||
| - **metadata**: arbitrary dict | - **metadata**: arbitrary dict | ||||
| - **indexer_configuration_id**: tool used to compute metadata | - **indexer_configuration_id**: tool used to compute metadata | ||||
| - **mappings** (List[str]): list of mappings used to translate | |||||
| these metadata | |||||
| conflict_update: Flag to determine if we want to overwrite (true) | conflict_update: Flag to determine if we want to overwrite (true) | ||||
| or skip duplicates (false, the default) | or skip duplicates (false, the default) | ||||
| """ | """ | ||||
| db.mktemp_origin_intrinsic_metadata(cur) | db.mktemp_origin_intrinsic_metadata(cur) | ||||
| db.copy_to(metadata, 'tmp_origin_intrinsic_metadata', | db.copy_to(metadata, 'tmp_origin_intrinsic_metadata', | ||||
| ['origin_id', 'metadata', 'indexer_configuration_id', | ['origin_id', 'metadata', 'indexer_configuration_id', | ||||
| 'from_revision'], | 'from_revision', 'mappings'], | ||||
| cur) | cur) | ||||
| db.origin_intrinsic_metadata_add_from_temp(conflict_update, cur) | db.origin_intrinsic_metadata_add_from_temp(conflict_update, cur) | ||||
| @remote_api_endpoint('origin_intrinsic_metadata/search/fulltext') | @remote_api_endpoint('origin_intrinsic_metadata/search/fulltext') | ||||
| @db_transaction_generator() | @db_transaction_generator() | ||||
| def origin_intrinsic_metadata_search_fulltext( | def origin_intrinsic_metadata_search_fulltext( | ||||
| self, conjunction, limit=100, db=None, cur=None): | self, conjunction, limit=100, db=None, cur=None): | ||||
| """Returns the list of origins whose metadata contain all the terms. | """Returns the list of origins whose metadata contain all the terms. | ||||
| Args: | Args: | ||||
| conjunction (List[str]): List of terms to be searched for. | conjunction (List[str]): List of terms to be searched for. | ||||
| limit (int): The maximum number of results to return | limit (int): The maximum number of results to return | ||||
| Yields: | Yields: | ||||
| list: dictionaries with the following keys: | list: dictionaries with the following keys: | ||||
| - **id** (int) | - **id** (int) | ||||
| - **metadata** (str): associated metadata | - **metadata** (str): associated metadata | ||||
| - **tool** (dict): tool used to compute metadata | - **tool** (dict): tool used to compute metadata | ||||
| - **mappings** (List[str]): list of mappings used to translate | |||||
| these metadata | |||||
| """ | """ | ||||
| for c in db.origin_intrinsic_metadata_search_fulltext( | for c in db.origin_intrinsic_metadata_search_fulltext( | ||||
| conjunction, limit=limit, cur=cur): | conjunction, limit=limit, cur=cur): | ||||
| yield converters.db_to_metadata( | yield converters.db_to_metadata( | ||||
| dict(zip(db.origin_intrinsic_metadata_cols, c))) | dict(zip(db.origin_intrinsic_metadata_cols, c))) | ||||
| @remote_api_endpoint('indexer_configuration/add') | @remote_api_endpoint('indexer_configuration/add') | ||||
| ▲ Show 20 Lines • Show All 55 Lines • Show Last 20 Lines | |||||
please remove those print statements.