Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/__init__.py
Show First 20 Lines • Show All 499 Lines • ▼ Show 20 Lines | def revision_metadata_add(self, metadata, conflict_update=False, db=None, | ||||
db.mktemp_revision_metadata(cur) | db.mktemp_revision_metadata(cur) | ||||
# empty metadata is mapped to 'unknown' | # empty metadata is mapped to 'unknown' | ||||
db.copy_to(metadata, 'tmp_revision_metadata', | db.copy_to(metadata, 'tmp_revision_metadata', | ||||
['id', 'translated_metadata', 'indexer_configuration_id'], | ['id', 'translated_metadata', 'indexer_configuration_id'], | ||||
cur) | cur) | ||||
db.revision_metadata_add_from_temp(conflict_update, cur) | db.revision_metadata_add_from_temp(conflict_update, cur) | ||||
@remote_api_endpoint('origin_intrinsic_metadata/missing') | |||||
@db_transaction_generator() | |||||
moranegg: is it necessary to know that the metadata for an origin is missing?
as we need to compute every… | |||||
Done Inline ActionsI don't know, but it can't hurt to have this endpoint available vlorentz: I don't know, but it can't hurt to have this endpoint available | |||||
def origin_intrinsic_metadata_missing(self, metadata, db=None, cur=None): | |||||
"""List metadata missing from storage. | |||||
Args: | |||||
metadata (iterable): dictionaries with keys: | |||||
Done Inline Actionsan origin can't be identified by sha1 moranegg: an origin can't be identified by sha1
is it the origin_id? | |||||
id (bytes): origin identifier | |||||
indexer_configuration_id (int): tool used to compute | |||||
the results | |||||
Returns: | |||||
iterable: missing ids | |||||
""" | |||||
for obj in db.origin_intrinsic_metadata_missing_from_list( | |||||
metadata, cur): | |||||
yield obj[0] | |||||
@remote_api_endpoint('origin_intrinsic_metadata') | |||||
@db_transaction_generator() | |||||
def origin_intrinsic_metadata_get(self, ids, db=None, cur=None): | |||||
"""Retrieve origin metadata per id. | |||||
Args: | |||||
ids (iterable): origin identifiers | |||||
Yields: | |||||
list: dictionaries with the following keys: | |||||
id (int) | |||||
translated_metadata (str): associated metadata | |||||
tool (dict): tool used to compute metadata | |||||
""" | |||||
for c in db.origin_intrinsic_metadata_get_from_list(ids, cur): | |||||
yield converters.db_to_metadata( | |||||
dict(zip(db.origin_intrinsic_metadata_cols, c))) | |||||
@remote_api_endpoint('origin_intrinsic_metadata/add') | |||||
@db_transaction() | |||||
def origin_intrinsic_metadata_add(self, metadata, | |||||
conflict_update=False, db=None, | |||||
cur=None): | |||||
"""Add metadata not present in storage. | |||||
Args: | |||||
Not Done Inline Actionssha1_git is not the origin identifier moranegg: sha1_git is not the origin identifier | |||||
metadata (iterable): dictionaries with keys: | |||||
- id: origin identifiers | |||||
- translated_metadata: bytes / jsonb ? | |||||
conflict_update: Flag to determine if we want to overwrite (true) | |||||
or skip duplicates (false, the default) | |||||
""" | |||||
db.mktemp_origin_intrinsic_metadata(cur) | |||||
# empty metadata is mapped to 'unknown' | |||||
db.copy_to(metadata, 'tmp_origin_intrinsic_metadata', | |||||
['id', 'translated_metadata', 'indexer_configuration_id'], | |||||
cur) | |||||
db.origin_intrinsic_metadata_add_from_temp(conflict_update, cur) | |||||
@remote_api_endpoint('indexer_configuration/add') | @remote_api_endpoint('indexer_configuration/add') | ||||
@db_transaction_generator() | @db_transaction_generator() | ||||
def indexer_configuration_add(self, tools, db=None, cur=None): | def indexer_configuration_add(self, tools, db=None, cur=None): | ||||
"""Add new tools to the storage. | """Add new tools to the storage. | ||||
Args: | Args: | ||||
tools ([dict]): List of dictionary representing tool to | tools ([dict]): List of dictionary representing tool to | ||||
insert in the db. Dictionary with the following keys:: | insert in the db. Dictionary with the following keys:: | ||||
▲ Show 20 Lines • Show All 48 Lines • Show Last 20 Lines |
is it necessary to know that the metadata for an origin is missing?
as we need to compute every time there is a new most_recent_HEAD_revision the metadata associated