Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/__init__.py
| Show All 24 Lines | |||||
| from .interface import PagedResult, Sha1 | from .interface import PagedResult, Sha1 | ||||
| from .metrics import process_metrics, send_metric, timed | from .metrics import process_metrics, send_metric, timed | ||||
| from .model import ( | from .model import ( | ||||
| ContentCtagsRow, | ContentCtagsRow, | ||||
| ContentLanguageRow, | ContentLanguageRow, | ||||
| ContentLicenseRow, | ContentLicenseRow, | ||||
| ContentMetadataRow, | ContentMetadataRow, | ||||
| ContentMimetypeRow, | ContentMimetypeRow, | ||||
| DirectoryIntrinsicMetadataRow, | |||||
| OriginIntrinsicMetadataRow, | OriginIntrinsicMetadataRow, | ||||
| RevisionIntrinsicMetadataRow, | |||||
| ) | ) | ||||
| from .writer import JournalWriter | from .writer import JournalWriter | ||||
| INDEXER_CFG_KEY = "indexer_storage" | INDEXER_CFG_KEY = "indexer_storage" | ||||
| MAPPING_NAMES = ["cff", "codemeta", "gemspec", "maven", "npm", "pkg-info"] | MAPPING_NAMES = ["cff", "codemeta", "gemspec", "maven", "npm", "pkg-info"] | ||||
| ▲ Show 20 Lines • Show All 474 Lines • ▼ Show 20 Lines | ) -> Dict[str, int]: | ||||
| ) | ) | ||||
| count = db.content_metadata_add_from_temp(cur) | count = db.content_metadata_add_from_temp(cur) | ||||
| return { | return { | ||||
| "content_metadata:add": count, | "content_metadata:add": count, | ||||
| } | } | ||||
| @timed | @timed | ||||
| @db_transaction() | @db_transaction() | ||||
| def revision_intrinsic_metadata_missing( | def directory_intrinsic_metadata_missing( | ||||
| self, metadata: Iterable[Dict], db=None, cur=None | self, metadata: Iterable[Dict], db=None, cur=None | ||||
| ) -> List[Tuple[Sha1, int]]: | ) -> List[Tuple[Sha1, int]]: | ||||
| return [ | return [ | ||||
| obj[0] | obj[0] | ||||
| for obj in db.revision_intrinsic_metadata_missing_from_list(metadata, cur) | for obj in db.directory_intrinsic_metadata_missing_from_list(metadata, cur) | ||||
| ] | ] | ||||
| @timed | @timed | ||||
| @db_transaction() | @db_transaction() | ||||
| def revision_intrinsic_metadata_get( | def directory_intrinsic_metadata_get( | ||||
| self, ids: Iterable[Sha1], db=None, cur=None | self, ids: Iterable[Sha1], db=None, cur=None | ||||
| ) -> List[RevisionIntrinsicMetadataRow]: | ) -> List[DirectoryIntrinsicMetadataRow]: | ||||
| return [ | return [ | ||||
| RevisionIntrinsicMetadataRow.from_dict( | DirectoryIntrinsicMetadataRow.from_dict( | ||||
| converters.db_to_metadata( | converters.db_to_metadata( | ||||
| dict(zip(db.revision_intrinsic_metadata_cols, c)) | dict(zip(db.directory_intrinsic_metadata_cols, c)) | ||||
| ) | ) | ||||
| ) | ) | ||||
| for c in db.revision_intrinsic_metadata_get_from_list(ids, cur) | for c in db.directory_intrinsic_metadata_get_from_list(ids, cur) | ||||
| ] | ] | ||||
| @timed | @timed | ||||
| @process_metrics | @process_metrics | ||||
| @db_transaction() | @db_transaction() | ||||
| def revision_intrinsic_metadata_add( | def directory_intrinsic_metadata_add( | ||||
| self, | self, | ||||
| metadata: List[RevisionIntrinsicMetadataRow], | metadata: List[DirectoryIntrinsicMetadataRow], | ||||
| db=None, | db=None, | ||||
| cur=None, | cur=None, | ||||
| ) -> Dict[str, int]: | ) -> Dict[str, int]: | ||||
| check_id_duplicates(metadata) | check_id_duplicates(metadata) | ||||
| metadata.sort(key=lambda m: m.id) | metadata.sort(key=lambda m: m.id) | ||||
| self.journal_writer.write_additions("revision_intrinsic_metadata", metadata) | self.journal_writer.write_additions("directory_intrinsic_metadata", metadata) | ||||
| db.mktemp_revision_intrinsic_metadata(cur) | db.mktemp_directory_intrinsic_metadata(cur) | ||||
| db.copy_to( | db.copy_to( | ||||
| [m.to_dict() for m in metadata], | [m.to_dict() for m in metadata], | ||||
| "tmp_revision_intrinsic_metadata", | "tmp_directory_intrinsic_metadata", | ||||
| ["id", "metadata", "mappings", "indexer_configuration_id"], | ["id", "metadata", "mappings", "indexer_configuration_id"], | ||||
| cur, | cur, | ||||
| ) | ) | ||||
| count = db.revision_intrinsic_metadata_add_from_temp(cur) | count = db.directory_intrinsic_metadata_add_from_temp(cur) | ||||
| return { | return { | ||||
| "revision_intrinsic_metadata:add": count, | "directory_intrinsic_metadata:add": count, | ||||
| } | } | ||||
| @timed | @timed | ||||
| @db_transaction() | @db_transaction() | ||||
| def origin_intrinsic_metadata_get( | def origin_intrinsic_metadata_get( | ||||
| self, urls: Iterable[str], db=None, cur=None | self, urls: Iterable[str], db=None, cur=None | ||||
| ) -> List[OriginIntrinsicMetadataRow]: | ) -> List[OriginIntrinsicMetadataRow]: | ||||
| return [ | return [ | ||||
| Show All 18 Lines | ) -> Dict[str, int]: | ||||
| metadata.sort(key=lambda m: m.id) | metadata.sort(key=lambda m: m.id) | ||||
| self.journal_writer.write_additions("origin_intrinsic_metadata", metadata) | self.journal_writer.write_additions("origin_intrinsic_metadata", metadata) | ||||
| db.mktemp_origin_intrinsic_metadata(cur) | db.mktemp_origin_intrinsic_metadata(cur) | ||||
| db.copy_to( | db.copy_to( | ||||
| [m.to_dict() for m in metadata], | [m.to_dict() for m in metadata], | ||||
| "tmp_origin_intrinsic_metadata", | "tmp_origin_intrinsic_metadata", | ||||
| ["id", "metadata", "indexer_configuration_id", "from_revision", "mappings"], | [ | ||||
| "id", | |||||
| "metadata", | |||||
| "indexer_configuration_id", | |||||
| "from_directory", | |||||
| "mappings", | |||||
| ], | |||||
| cur, | cur, | ||||
| ) | ) | ||||
| count = db.origin_intrinsic_metadata_add_from_temp(cur) | count = db.origin_intrinsic_metadata_add_from_temp(cur) | ||||
| return { | return { | ||||
| "origin_intrinsic_metadata:add": count, | "origin_intrinsic_metadata:add": count, | ||||
| } | } | ||||
| @timed | @timed | ||||
| ▲ Show 20 Lines • Show All 138 Lines • Show Last 20 Lines | |||||