Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/__init__.py
| Show First 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | def check_id_duplicates(data): | ||||
| duplicates = [id_ for (id_, count) in counter.items() if count >= 2] | duplicates = [id_ for (id_, count) in counter.items() if count >= 2] | ||||
| if duplicates: | if duplicates: | ||||
| raise DuplicateId(list(map(dict, duplicates))) | raise DuplicateId(list(map(dict, duplicates))) | ||||
| class IndexerStorage: | class IndexerStorage: | ||||
| """SWH Indexer Storage Datastore""" | """SWH Indexer Storage Datastore""" | ||||
| current_version = 135 | current_version = 136 | ||||
| def __init__(self, db, min_pool_conns=1, max_pool_conns=10, journal_writer=None): | def __init__(self, db, min_pool_conns=1, max_pool_conns=10, journal_writer=None): | ||||
| """ | """ | ||||
| Args: | Args: | ||||
| db: either a libpq connection string, or a psycopg2 connection | db: either a libpq connection string, or a psycopg2 connection | ||||
| journal_writer: configuration passed to | journal_writer: configuration passed to | ||||
| `swh.journal.writer.get_journal_writer` | `swh.journal.writer.get_journal_writer` | ||||
| ▲ Show 20 Lines • Show All 140 Lines • ▼ Show 20 Lines | class IndexerStorage: | ||||
| @db_transaction() | @db_transaction() | ||||
| def content_mimetype_add( | def content_mimetype_add( | ||||
| self, | self, | ||||
| mimetypes: List[ContentMimetypeRow], | mimetypes: List[ContentMimetypeRow], | ||||
| db=None, | db=None, | ||||
| cur=None, | cur=None, | ||||
| ) -> Dict[str, int]: | ) -> Dict[str, int]: | ||||
| check_id_duplicates(mimetypes) | check_id_duplicates(mimetypes) | ||||
| mimetypes.sort(key=lambda m: m.id) | |||||
| self.journal_writer.write_additions("content_mimetype", mimetypes) | self.journal_writer.write_additions("content_mimetype", mimetypes) | ||||
| db.mktemp_content_mimetype(cur) | db.mktemp_content_mimetype(cur) | ||||
| db.copy_to( | db.copy_to( | ||||
| [m.to_dict() for m in mimetypes], | [m.to_dict() for m in mimetypes], | ||||
| "tmp_content_mimetype", | "tmp_content_mimetype", | ||||
| ["id", "mimetype", "encoding", "indexer_configuration_id"], | ["id", "mimetype", "encoding", "indexer_configuration_id"], | ||||
| cur, | cur, | ||||
| ) | ) | ||||
| Show All 31 Lines | class IndexerStorage: | ||||
| @db_transaction() | @db_transaction() | ||||
| def content_fossology_license_add( | def content_fossology_license_add( | ||||
| self, | self, | ||||
| licenses: List[ContentLicenseRow], | licenses: List[ContentLicenseRow], | ||||
| db=None, | db=None, | ||||
| cur=None, | cur=None, | ||||
| ) -> Dict[str, int]: | ) -> Dict[str, int]: | ||||
| check_id_duplicates(licenses) | check_id_duplicates(licenses) | ||||
| licenses.sort(key=lambda m: m.id) | |||||
| self.journal_writer.write_additions("content_fossology_license", licenses) | self.journal_writer.write_additions("content_fossology_license", licenses) | ||||
| db.mktemp_content_fossology_license(cur) | db.mktemp_content_fossology_license(cur) | ||||
| db.copy_to( | db.copy_to( | ||||
| [license.to_dict() for license in licenses], | [license.to_dict() for license in licenses], | ||||
| tblname="tmp_content_fossology_license", | tblname="tmp_content_fossology_license", | ||||
| columns=["id", "license", "indexer_configuration_id"], | columns=["id", "license", "indexer_configuration_id"], | ||||
| cur=cur, | cur=cur, | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | class IndexerStorage: | ||||
| @db_transaction() | @db_transaction() | ||||
| def content_metadata_add( | def content_metadata_add( | ||||
| self, | self, | ||||
| metadata: List[ContentMetadataRow], | metadata: List[ContentMetadataRow], | ||||
| db=None, | db=None, | ||||
| cur=None, | cur=None, | ||||
| ) -> Dict[str, int]: | ) -> Dict[str, int]: | ||||
| check_id_duplicates(metadata) | check_id_duplicates(metadata) | ||||
| metadata.sort(key=lambda m: m.id) | |||||
| self.journal_writer.write_additions("content_metadata", metadata) | self.journal_writer.write_additions("content_metadata", metadata) | ||||
| db.mktemp_content_metadata(cur) | db.mktemp_content_metadata(cur) | ||||
| rows = [m.to_dict() for m in metadata] | rows = [m.to_dict() for m in metadata] | ||||
| for row in rows: | for row in rows: | ||||
| row["metadata"] = sanitize_json(row["metadata"]) | row["metadata"] = sanitize_json(row["metadata"]) | ||||
| Show All 37 Lines | class IndexerStorage: | ||||
| @db_transaction() | @db_transaction() | ||||
| def directory_intrinsic_metadata_add( | def directory_intrinsic_metadata_add( | ||||
| self, | self, | ||||
| metadata: List[DirectoryIntrinsicMetadataRow], | metadata: List[DirectoryIntrinsicMetadataRow], | ||||
| db=None, | db=None, | ||||
| cur=None, | cur=None, | ||||
| ) -> Dict[str, int]: | ) -> Dict[str, int]: | ||||
| check_id_duplicates(metadata) | check_id_duplicates(metadata) | ||||
| metadata.sort(key=lambda m: m.id) | |||||
| self.journal_writer.write_additions("directory_intrinsic_metadata", metadata) | self.journal_writer.write_additions("directory_intrinsic_metadata", metadata) | ||||
| db.mktemp_directory_intrinsic_metadata(cur) | db.mktemp_directory_intrinsic_metadata(cur) | ||||
| rows = [m.to_dict() for m in metadata] | rows = [m.to_dict() for m in metadata] | ||||
| for row in rows: | for row in rows: | ||||
| row["metadata"] = sanitize_json(row["metadata"]) | row["metadata"] = sanitize_json(row["metadata"]) | ||||
| Show All 27 Lines | class IndexerStorage: | ||||
| @db_transaction() | @db_transaction() | ||||
| def origin_intrinsic_metadata_add( | def origin_intrinsic_metadata_add( | ||||
| self, | self, | ||||
| metadata: List[OriginIntrinsicMetadataRow], | metadata: List[OriginIntrinsicMetadataRow], | ||||
| db=None, | db=None, | ||||
| cur=None, | cur=None, | ||||
| ) -> Dict[str, int]: | ) -> Dict[str, int]: | ||||
| check_id_duplicates(metadata) | check_id_duplicates(metadata) | ||||
| metadata.sort(key=lambda m: m.id) | |||||
| self.journal_writer.write_additions("origin_intrinsic_metadata", metadata) | self.journal_writer.write_additions("origin_intrinsic_metadata", metadata) | ||||
| db.mktemp_origin_intrinsic_metadata(cur) | db.mktemp_origin_intrinsic_metadata(cur) | ||||
| rows = [m.to_dict() for m in metadata] | rows = [m.to_dict() for m in metadata] | ||||
| for row in rows: | for row in rows: | ||||
| row["metadata"] = sanitize_json(row["metadata"]) | row["metadata"] = sanitize_json(row["metadata"]) | ||||
| ▲ Show 20 Lines • Show All 125 Lines • ▼ Show 20 Lines | class IndexerStorage: | ||||
| @db_transaction() | @db_transaction() | ||||
| def origin_extrinsic_metadata_add( | def origin_extrinsic_metadata_add( | ||||
| self, | self, | ||||
| metadata: List[OriginExtrinsicMetadataRow], | metadata: List[OriginExtrinsicMetadataRow], | ||||
| db=None, | db=None, | ||||
| cur=None, | cur=None, | ||||
| ) -> Dict[str, int]: | ) -> Dict[str, int]: | ||||
| check_id_duplicates(metadata) | check_id_duplicates(metadata) | ||||
| metadata.sort(key=lambda m: m.id) | |||||
| self.journal_writer.write_additions("origin_extrinsic_metadata", metadata) | self.journal_writer.write_additions("origin_extrinsic_metadata", metadata) | ||||
| db.mktemp_origin_extrinsic_metadata(cur) | db.mktemp_origin_extrinsic_metadata(cur) | ||||
| rows = [m.to_dict() for m in metadata] | rows = [m.to_dict() for m in metadata] | ||||
| for row in rows: | for row in rows: | ||||
| row["metadata"] = sanitize_json(row["metadata"]) | row["metadata"] = sanitize_json(row["metadata"]) | ||||
| ▲ Show 20 Lines • Show All 64 Lines • Show Last 20 Lines | |||||