Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/__init__.py
| Show First 20 Lines • Show All 525 Lines • ▼ Show 20 Lines | def content_fossology_license_get_range( | ||||
| - **next** (Optional[bytes]): The next range of sha1 starts at | - **next** (Optional[bytes]): The next range of sha1 starts at | ||||
| this sha1 if any | this sha1 if any | ||||
| """ | """ | ||||
| return self._content_get_range('fossology_license', start, end, | return self._content_get_range('fossology_license', start, end, | ||||
| indexer_configuration_id, limit=limit, | indexer_configuration_id, limit=limit, | ||||
| with_textual_data=True, db=db, cur=cur) | with_textual_data=True, db=db, cur=cur) | ||||
| @remote_api_endpoint('content_metadata/missing') | |||||
| @db_transaction_generator() | |||||
| def content_metadata_missing(self, metadata, db=None, cur=None): | |||||
| """List metadata missing from storage. | |||||
| Args: | |||||
| metadata (iterable): dictionaries with keys: | |||||
| - **id** (bytes): sha1 identifier | |||||
| - **indexer_configuration_id** (int): tool used to compute | |||||
| the results | |||||
| Yields: | |||||
| missing sha1s | |||||
| """ | |||||
| for obj in db.content_metadata_missing_from_list(metadata, cur): | |||||
| yield obj[0] | |||||
| @remote_api_endpoint('content_metadata') | |||||
| @db_transaction_generator() | |||||
| def content_metadata_get(self, ids, db=None, cur=None): | |||||
| """Retrieve metadata per id. | |||||
| Args: | |||||
| ids (iterable): sha1 checksums | |||||
| Yields: | |||||
| dictionaries with the following keys: | |||||
| id (bytes) | |||||
| metadata (str): associated metadata | |||||
| tool (dict): tool used to compute metadata | |||||
| """ | |||||
| for c in db.content_metadata_get_from_list(ids, cur): | |||||
| yield converters.db_to_metadata( | |||||
| dict(zip(db.content_metadata_cols, c))) | |||||
| @remote_api_endpoint('content_metadata/add') | |||||
| @db_transaction() | |||||
| def content_metadata_add(self, metadata, conflict_update=False, db=None, | |||||
| cur=None): | |||||
| """Add metadata not present in storage. | |||||
| Args: | |||||
| metadata (iterable): dictionaries with keys: | |||||
| - **id**: sha1 | |||||
| - **metadata**: arbitrary dict | |||||
| conflict_update: Flag to determine if we want to overwrite (true) | |||||
| or skip duplicates (false, the default) | |||||
| """ | |||||
| _check_id_duplicates(metadata) | |||||
| metadata.sort(key=lambda m: m['id']) | |||||
| db.mktemp_content_metadata(cur) | |||||
| db.copy_to(metadata, 'tmp_content_metadata', | |||||
| ['id', 'metadata', 'indexer_configuration_id'], | |||||
| cur) | |||||
| db.content_metadata_add_from_temp(conflict_update, cur) | |||||
| @remote_api_endpoint('revision_intrinsic_metadata/missing') | @remote_api_endpoint('revision_intrinsic_metadata/missing') | ||||
| @db_transaction_generator() | @db_transaction_generator() | ||||
| def revision_intrinsic_metadata_missing(self, metadata, db=None, cur=None): | def revision_intrinsic_metadata_missing(self, metadata, db=None, cur=None): | ||||
| """List metadata missing from storage. | """List metadata missing from storage. | ||||
| Args: | Args: | ||||
| metadata (iterable): dictionaries with keys: | metadata (iterable): dictionaries with keys: | ||||
| ▲ Show 20 Lines • Show All 322 Lines • Show Last 20 Lines | |||||