Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/storage.py
Show First 20 Lines • Show All 1,737 Lines • ▼ Show 20 Lines | |||||
@db_transaction() | @db_transaction() | ||||
def origin_metadata_add(self, origin_id, ts, provider, tool, metadata, | def origin_metadata_add(self, origin_id, ts, provider, tool, metadata, | ||||
db=None, cur=None): | db=None, cur=None): | ||||
""" Add an origin_metadata for the origin at ts with provenance and | """ Add an origin_metadata for the origin at ts with provenance and | ||||
metadata. | metadata. | ||||
Args: | Args: | ||||
origin_id (int): the origin's id for which the metadata is added | origin_id (str): the origin URL for which the metadata is added | ||||
ts (datetime): timestamp of the found metadata | ts (datetime): timestamp of the found metadata | ||||
provider (int): the provider of metadata (ex:'hal') | provider (int): the provider of metadata (ex:'hal') | ||||
tool (int): tool used to extract metadata | tool (int): tool used to extract metadata | ||||
metadata (jsonb): the metadata retrieved at the time and location | metadata (jsonb): the metadata retrieved at the time and location | ||||
Returns: | Returns: | ||||
id (int): the origin_metadata unique id | id (int): the origin_metadata unique id | ||||
""" | """ | ||||
if isinstance(origin_id, str): | |||||
origin = self.origin_get({'url': origin_id}, db=db, cur=cur) | origin = self.origin_get({'url': origin_id}, db=db, cur=cur) | ||||
if not origin: | if not origin: | ||||
return | return | ||||
origin_id = origin['id'] | origin_id = origin['id'] | ||||
if isinstance(ts, str): | if isinstance(ts, str): | ||||
ts = dateutil.parser.parse(ts) | ts = dateutil.parser.parse(ts) | ||||
return db.origin_metadata_add(origin_id, ts, provider, tool, | return db.origin_metadata_add(origin_id, ts, provider, tool, | ||||
metadata, cur) | metadata, cur) | ||||
@db_transaction_generator(statement_timeout=500) | @db_transaction_generator(statement_timeout=500) | ||||
def origin_metadata_get_by(self, origin_id, provider_type=None, db=None, | def origin_metadata_get_by(self, origin_id, provider_type=None, db=None, | ||||
cur=None): | cur=None): | ||||
"""Retrieve list of all origin_metadata entries for the origin_id | """Retrieve list of all origin_metadata entries for the origin_id | ||||
Args: | Args: | ||||
origin_id (int): the unique origin identifier | origin_id (str): the origin URL | ||||
provider_type (str): (optional) type of provider | provider_type (str): (optional) type of provider | ||||
Returns: | Returns: | ||||
list of dicts: the origin_metadata dictionary with the keys: | list of dicts: the origin_metadata dictionary with the keys: | ||||
- origin_id (int): origin's id | - origin_id (str): origin's URL | ||||
- discovery_date (datetime): timestamp of discovery | - discovery_date (datetime): timestamp of discovery | ||||
- tool_id (int): metadata's extracting tool | - tool_id (int): metadata's extracting tool | ||||
- metadata (jsonb) | - metadata (jsonb) | ||||
- provider_id (int): metadata's provider | - provider_id (int): metadata's provider | ||||
- provider_name (str) | - provider_name (str) | ||||
- provider_type (str) | - provider_type (str) | ||||
- provider_url (str) | - provider_url (str) | ||||
""" | """ | ||||
if isinstance(origin_id, str): | |||||
origin = self.origin_get({'url': origin_id}, db=db, cur=cur) | origin = self.origin_get({'url': origin_id}, db=db, cur=cur) | ||||
if not origin: | if not origin: | ||||
return | return | ||||
origin_id = origin['id'] | origin_url = origin['url'] | ||||
for line in db.origin_metadata_get_by(origin_id, provider_type, cur): | for line in db.origin_metadata_get_by(origin_url, provider_type, cur): | ||||
yield dict(zip(db.origin_metadata_get_cols, line)) | yield dict(zip(db.origin_metadata_get_cols, line)) | ||||
@db_transaction() | @db_transaction() | ||||
def tool_add(self, tools, db=None, cur=None): | def tool_add(self, tools, db=None, cur=None): | ||||
"""Add new tools to the storage. | """Add new tools to the storage. | ||||
Args: | Args: | ||||
tools (iterable of :class:`dict`): Tool information to add to | tools (iterable of :class:`dict`): Tool information to add to | ||||
▲ Show 20 Lines • Show All 145 Lines • Show Last 20 Lines |