Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/__init__.py
Show First 20 Lines • Show All 589 Lines • ▼ Show 20 Lines | def content_metadata_add(self, metadata, conflict_update=False, db=None, | ||||
db.mktemp_content_metadata(cur) | db.mktemp_content_metadata(cur) | ||||
db.copy_to(metadata, 'tmp_content_metadata', | db.copy_to(metadata, 'tmp_content_metadata', | ||||
['id', 'translated_metadata', 'indexer_configuration_id'], | ['id', 'translated_metadata', 'indexer_configuration_id'], | ||||
cur) | cur) | ||||
db.content_metadata_add_from_temp(conflict_update, cur) | db.content_metadata_add_from_temp(conflict_update, cur) | ||||
@remote_api_endpoint('revision_metadata/missing') | @remote_api_endpoint('revision_intrinsic_metadata/missing') | ||||
@db_transaction_generator() | @db_transaction_generator() | ||||
def revision_metadata_missing(self, metadata, db=None, cur=None): | def revision_intrinsic_metadata_missing(self, metadata, db=None, cur=None): | ||||
"""List metadata missing from storage. | """List metadata missing from storage. | ||||
Args: | Args: | ||||
metadata (iterable): dictionaries with keys: | metadata (iterable): dictionaries with keys: | ||||
- **id** (bytes): sha1_git revision identifier | - **id** (bytes): sha1_git revision identifier | ||||
- **indexer_configuration_id** (int): tool used to compute | - **indexer_configuration_id** (int): tool used to compute | ||||
the results | the results | ||||
Yields: | Yields: | ||||
missing ids | missing ids | ||||
""" | """ | ||||
for obj in db.revision_metadata_missing_from_list(metadata, cur): | for obj in db.revision_intrinsic_metadata_missing_from_list( | ||||
metadata, cur): | |||||
yield obj[0] | yield obj[0] | ||||
@remote_api_endpoint('revision_metadata') | @remote_api_endpoint('revision_intrinsic_metadata') | ||||
@db_transaction_generator() | @db_transaction_generator() | ||||
def revision_metadata_get(self, ids, db=None, cur=None): | def revision_intrinsic_metadata_get(self, ids, db=None, cur=None): | ||||
"""Retrieve revision metadata per id. | """Retrieve revision metadata per id. | ||||
Args: | Args: | ||||
ids (iterable): sha1 checksums | ids (iterable): sha1 checksums | ||||
Yields: | Yields: | ||||
dictionaries with the following keys: | dictionaries with the following keys: | ||||
- **id** (bytes) | - **id** (bytes) | ||||
- **translated_metadata** (str): associated metadata | - **metadata** (str): associated metadata | ||||
- **tool** (dict): tool used to compute metadata | - **tool** (dict): tool used to compute metadata | ||||
- **mappings** (List[str]): list of mappings used to translate | - **mappings** (List[str]): list of mappings used to translate | ||||
these metadata | these metadata | ||||
""" | """ | ||||
for c in db.revision_metadata_get_from_list(ids, cur): | for c in db.revision_intrinsic_metadata_get_from_list(ids, cur): | ||||
yield converters.db_to_metadata( | yield converters.db_to_metadata( | ||||
dict(zip(db.revision_metadata_cols, c))) | dict(zip(db.revision_intrinsic_metadata_cols, c))) | ||||
@remote_api_endpoint('revision_metadata/add') | @remote_api_endpoint('revision_intrinsic_metadata/add') | ||||
@db_transaction() | @db_transaction() | ||||
def revision_metadata_add(self, metadata, conflict_update=False, db=None, | def revision_intrinsic_metadata_add(self, metadata, conflict_update=False, | ||||
cur=None): | db=None, cur=None): | ||||
"""Add metadata not present in storage. | """Add metadata not present in storage. | ||||
Args: | Args: | ||||
metadata (iterable): dictionaries with keys: | metadata (iterable): dictionaries with keys: | ||||
- **id**: sha1_git of revision | - **id**: sha1_git of revision | ||||
- **translated_metadata**: arbitrary dict | - **metadata**: arbitrary dict | ||||
- **indexer_configuration_id**: tool used to compute metadata | - **indexer_configuration_id**: tool used to compute metadata | ||||
- **mappings** (List[str]): list of mappings used to translate | - **mappings** (List[str]): list of mappings used to translate | ||||
these metadata | these metadata | ||||
conflict_update: Flag to determine if we want to overwrite (true) | conflict_update: Flag to determine if we want to overwrite (true) | ||||
or skip duplicates (false, the default) | or skip duplicates (false, the default) | ||||
""" | """ | ||||
_check_duplicates(metadata, 'id') | _check_duplicates(metadata, 'id') | ||||
metadata.sort(key=lambda m: m['id']) | metadata.sort(key=lambda m: m['id']) | ||||
db.mktemp_revision_metadata(cur) | db.mktemp_revision_intrinsic_metadata(cur) | ||||
db.copy_to(metadata, 'tmp_revision_metadata', | db.copy_to(metadata, 'tmp_revision_intrinsic_metadata', | ||||
['id', 'translated_metadata', 'mappings', | ['id', 'metadata', 'mappings', | ||||
'indexer_configuration_id'], | 'indexer_configuration_id'], | ||||
cur) | cur) | ||||
db.revision_metadata_add_from_temp(conflict_update, cur) | db.revision_intrinsic_metadata_add_from_temp(conflict_update, cur) | ||||
@remote_api_endpoint('revision_metadata/delete') | @remote_api_endpoint('revision_intrinsic_metadata/delete') | ||||
@db_transaction() | @db_transaction() | ||||
def revision_metadata_delete(self, entries, db=None, cur=None): | def revision_intrinsic_metadata_delete(self, entries, db=None, cur=None): | ||||
"""Remove revision metadata from the storage. | """Remove revision metadata from the storage. | ||||
Args: | Args: | ||||
entries (dict): dictionaries with the following keys: | entries (dict): dictionaries with the following keys: | ||||
- **id** (bytes): revision identifier | - **id** (bytes): revision identifier | ||||
- **indexer_configuration_id** (int): tool used to compute | - **indexer_configuration_id** (int): tool used to compute | ||||
metadata | metadata | ||||
""" | """ | ||||
db.revision_metadata_delete(entries, cur) | db.revision_intrinsic_metadata_delete(entries, cur) | ||||
@remote_api_endpoint('origin_intrinsic_metadata') | @remote_api_endpoint('origin_intrinsic_metadata') | ||||
@db_transaction_generator() | @db_transaction_generator() | ||||
def origin_intrinsic_metadata_get(self, ids, db=None, cur=None): | def origin_intrinsic_metadata_get(self, ids, db=None, cur=None): | ||||
"""Retrieve origin metadata per id. | """Retrieve origin metadata per id. | ||||
Args: | Args: | ||||
ids (iterable): origin identifiers | ids (iterable): origin identifiers | ||||
Yields: | Yields: | ||||
list: dictionaries with the following keys: | list: dictionaries with the following keys: | ||||
- **origin_id** (int) | - **id** (int) | ||||
- **metadata** (str): associated metadata | - **metadata** (str): associated metadata | ||||
- **tool** (dict): tool used to compute metadata | - **tool** (dict): tool used to compute metadata | ||||
- **mappings** (List[str]): list of mappings used to translate | - **mappings** (List[str]): list of mappings used to translate | ||||
these metadata | these metadata | ||||
""" | """ | ||||
for c in db.origin_intrinsic_metadata_get_from_list(ids, cur): | for c in db.origin_intrinsic_metadata_get_from_list(ids, cur): | ||||
yield converters.db_to_metadata( | yield converters.db_to_metadata( | ||||
dict(zip(db.origin_intrinsic_metadata_cols, c))) | dict(zip(db.origin_intrinsic_metadata_cols, c))) | ||||
@remote_api_endpoint('origin_intrinsic_metadata/add') | @remote_api_endpoint('origin_intrinsic_metadata/add') | ||||
@db_transaction() | @db_transaction() | ||||
def origin_intrinsic_metadata_add(self, metadata, | def origin_intrinsic_metadata_add(self, metadata, | ||||
conflict_update=False, db=None, | conflict_update=False, db=None, | ||||
cur=None): | cur=None): | ||||
"""Add origin metadata not present in storage. | """Add origin metadata not present in storage. | ||||
Args: | Args: | ||||
metadata (iterable): dictionaries with keys: | metadata (iterable): dictionaries with keys: | ||||
- **origin_id**: origin identifier | - **id**: origin identifier | ||||
- **from_revision**: sha1 id of the revision used to generate | - **from_revision**: sha1 id of the revision used to generate | ||||
these metadata. | these metadata. | ||||
- **metadata**: arbitrary dict | - **metadata**: arbitrary dict | ||||
- **indexer_configuration_id**: tool used to compute metadata | - **indexer_configuration_id**: tool used to compute metadata | ||||
- **mappings** (List[str]): list of mappings used to translate | - **mappings** (List[str]): list of mappings used to translate | ||||
these metadata | these metadata | ||||
conflict_update: Flag to determine if we want to overwrite (true) | conflict_update: Flag to determine if we want to overwrite (true) | ||||
or skip duplicates (false, the default) | or skip duplicates (false, the default) | ||||
""" | """ | ||||
_check_duplicates(metadata, 'origin_id') | _check_duplicates(metadata, 'id') | ||||
metadata.sort(key=lambda m: m['origin_id']) | metadata.sort(key=lambda m: m['id']) | ||||
vlorentz: You can change `_check_duplicates` to remove the second argument, it's no longer needed now. | |||||
db.mktemp_origin_intrinsic_metadata(cur) | db.mktemp_origin_intrinsic_metadata(cur) | ||||
db.copy_to(metadata, 'tmp_origin_intrinsic_metadata', | db.copy_to(metadata, 'tmp_origin_intrinsic_metadata', | ||||
['origin_id', 'metadata', 'indexer_configuration_id', | ['id', 'metadata', 'indexer_configuration_id', | ||||
'from_revision', 'mappings'], | 'from_revision', 'mappings'], | ||||
cur) | cur) | ||||
db.origin_intrinsic_metadata_add_from_temp(conflict_update, cur) | db.origin_intrinsic_metadata_add_from_temp(conflict_update, cur) | ||||
@remote_api_endpoint('origin_intrinsic_metadata/delete') | @remote_api_endpoint('origin_intrinsic_metadata/delete') | ||||
@db_transaction() | @db_transaction() | ||||
def origin_intrinsic_metadata_delete( | def origin_intrinsic_metadata_delete( | ||||
self, entries, db=None, cur=None): | self, entries, db=None, cur=None): | ||||
▲ Show 20 Lines • Show All 175 Lines • Show Last 20 Lines |
You can change _check_duplicates to remove the second argument, it's no longer needed now.