Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/__init__.py
| # Copyright (C) 2015-2020 The Software Heritage developers | # Copyright (C) 2015-2022 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from collections import Counter | from collections import Counter | ||||
| from importlib import import_module | from importlib import import_module | ||||
| import json | import json | ||||
| from typing import Dict, Iterable, List, Optional, Tuple, Union | from typing import Dict, Iterable, List, Optional, Tuple, Union | ||||
| ▲ Show 20 Lines • Show All 105 Lines • ▼ Show 20 Lines | def check_id_duplicates(data): | ||||
| """ # noqa | """ # noqa | ||||
| counter = Counter(tuple(sorted(item.unique_key().items())) for item in data) | counter = Counter(tuple(sorted(item.unique_key().items())) for item in data) | ||||
| duplicates = [id_ for (id_, count) in counter.items() if count >= 2] | duplicates = [id_ for (id_, count) in counter.items() if count >= 2] | ||||
| if duplicates: | if duplicates: | ||||
| raise DuplicateId(list(map(dict, duplicates))) | raise DuplicateId(list(map(dict, duplicates))) | ||||
| class IndexerStorage: | class IndexerStorage: | ||||
| """SWH Indexer Storage""" | """SWH Indexer Storage Datastore""" | ||||
| current_version = 134 | |||||
| def __init__(self, db, min_pool_conns=1, max_pool_conns=10, journal_writer=None): | def __init__(self, db, min_pool_conns=1, max_pool_conns=10, journal_writer=None): | ||||
| """ | """ | ||||
| Args: | Args: | ||||
| db: either a libpq connection string, or a psycopg2 connection | db: either a libpq connection string, or a psycopg2 connection | ||||
| journal_writer: configuration passed to | journal_writer: configuration passed to | ||||
| `swh.journal.writer.get_journal_writer` | `swh.journal.writer.get_journal_writer` | ||||
| Show All 15 Lines | def get_db(self): | ||||
| if self._db: | if self._db: | ||||
| return self._db | return self._db | ||||
| return Db.from_pool(self._pool) | return Db.from_pool(self._pool) | ||||
| def put_db(self, db): | def put_db(self, db): | ||||
| if db is not self._db: | if db is not self._db: | ||||
| db.put_conn() | db.put_conn() | ||||
| @db_transaction() | |||||
| def get_current_version(self, *, db=None, cur=None): | |||||
| return db.current_version | |||||
| @timed | @timed | ||||
| @db_transaction() | @db_transaction() | ||||
| def check_config(self, *, check_write, db=None, cur=None): | def check_config(self, *, check_write, db=None, cur=None): | ||||
| # Check permissions on one of the tables | # Check permissions on one of the tables | ||||
| if check_write: | if check_write: | ||||
| check = "INSERT" | check = "INSERT" | ||||
| else: | else: | ||||
| check = "SELECT" | check = "SELECT" | ||||
| ▲ Show 20 Lines • Show All 591 Lines • Show Last 20 Lines | |||||