Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/__init__.py
Show All 19 Lines | |||||
from swh.storage.utils import get_partition_bounds_bytes | from swh.storage.utils import get_partition_bounds_bytes | ||||
from . import converters | from . import converters | ||||
from .db import Db | from .db import Db | ||||
from .exc import DuplicateId, IndexerStorageArgumentException | from .exc import DuplicateId, IndexerStorageArgumentException | ||||
from .interface import PagedResult, Sha1 | from .interface import PagedResult, Sha1 | ||||
from .metrics import process_metrics, send_metric, timed | from .metrics import process_metrics, send_metric, timed | ||||
from .model import ( | from .model import ( | ||||
ContentCtagsRow, | |||||
ContentLanguageRow, | ContentLanguageRow, | ||||
ContentLicenseRow, | ContentLicenseRow, | ||||
ContentMetadataRow, | ContentMetadataRow, | ||||
ContentMimetypeRow, | ContentMimetypeRow, | ||||
DirectoryIntrinsicMetadataRow, | DirectoryIntrinsicMetadataRow, | ||||
OriginExtrinsicMetadataRow, | OriginExtrinsicMetadataRow, | ||||
OriginIntrinsicMetadataRow, | OriginIntrinsicMetadataRow, | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 314 Lines • ▼ Show 20 Lines | ) -> Dict[str, int]: | ||||
cur, | cur, | ||||
) | ) | ||||
count = db.content_language_add_from_temp(cur) | count = db.content_language_add_from_temp(cur) | ||||
return {"content_language:add": count} | return {"content_language:add": count} | ||||
@timed | @timed | ||||
@db_transaction() | @db_transaction() | ||||
def content_ctags_missing( | |||||
self, ctags: Iterable[Dict], db=None, cur=None | |||||
) -> List[Tuple[Sha1, int]]: | |||||
return [obj[0] for obj in db.content_ctags_missing_from_list(ctags, cur)] | |||||
@timed | |||||
@db_transaction() | |||||
def content_ctags_get( | |||||
self, ids: Iterable[Sha1], db=None, cur=None | |||||
) -> List[ContentCtagsRow]: | |||||
return [ | |||||
ContentCtagsRow.from_dict( | |||||
converters.db_to_ctags(dict(zip(db.content_ctags_cols, c))) | |||||
) | |||||
for c in db.content_ctags_get_from_list(ids, cur) | |||||
] | |||||
@timed | |||||
@process_metrics | |||||
@db_transaction() | |||||
def content_ctags_add( | |||||
self, | |||||
ctags: List[ContentCtagsRow], | |||||
db=None, | |||||
cur=None, | |||||
) -> Dict[str, int]: | |||||
check_id_duplicates(ctags) | |||||
ctags.sort(key=lambda m: m.id) | |||||
self.journal_writer.write_additions("content_ctags", ctags) | |||||
db.mktemp_content_ctags(cur) | |||||
db.copy_to( | |||||
[ctag.to_dict() for ctag in ctags], | |||||
tblname="tmp_content_ctags", | |||||
columns=["id", "name", "kind", "line", "lang", "indexer_configuration_id"], | |||||
cur=cur, | |||||
) | |||||
count = db.content_ctags_add_from_temp(cur) | |||||
return {"content_ctags:add": count} | |||||
@timed | |||||
@db_transaction() | |||||
def content_ctags_search( | |||||
self, | |||||
expression: str, | |||||
limit: int = 10, | |||||
last_sha1: Optional[Sha1] = None, | |||||
db=None, | |||||
cur=None, | |||||
) -> List[ContentCtagsRow]: | |||||
return [ | |||||
ContentCtagsRow.from_dict( | |||||
converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj))) | |||||
) | |||||
for obj in db.content_ctags_search(expression, last_sha1, limit, cur=cur) | |||||
] | |||||
@timed | |||||
@db_transaction() | |||||
def content_fossology_license_get( | def content_fossology_license_get( | ||||
self, ids: Iterable[Sha1], db=None, cur=None | self, ids: Iterable[Sha1], db=None, cur=None | ||||
) -> List[ContentLicenseRow]: | ) -> List[ContentLicenseRow]: | ||||
return [ | return [ | ||||
ContentLicenseRow.from_dict( | ContentLicenseRow.from_dict( | ||||
converters.db_to_fossology_license( | converters.db_to_fossology_license( | ||||
dict(zip(db.content_fossology_license_cols, c)) | dict(zip(db.content_fossology_license_cols, c)) | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 376 Lines • Show Last 20 Lines |