Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/postgresql/storage.py
| # Copyright (C) 2015-2021 The Software Heritage developers | # Copyright (C) 2015-2021 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import base64 | import base64 | ||||
| from collections import defaultdict | from collections import defaultdict | ||||
| import contextlib | import contextlib | ||||
| from contextlib import contextmanager | from contextlib import contextmanager | ||||
| import datetime | import datetime | ||||
| import itertools | import itertools | ||||
| import operator | |||||
| from typing import Any, Counter, Dict, Iterable, List, Optional, Sequence, Tuple | from typing import Any, Counter, Dict, Iterable, List, Optional, Sequence, Tuple | ||||
| import attr | import attr | ||||
| import psycopg2 | import psycopg2 | ||||
| import psycopg2.errors | import psycopg2.errors | ||||
| import psycopg2.pool | import psycopg2.pool | ||||
| from swh.core.api.serializers import msgpack_dumps, msgpack_loads | from swh.core.api.serializers import msgpack_dumps, msgpack_loads | ||||
| ▲ Show 20 Lines • Show All 294 Lines • ▼ Show 20 Lines | ) -> PagedResult[Content]: | ||||
| contents.append(content) | contents.append(content) | ||||
| assert len(contents) <= limit | assert len(contents) <= limit | ||||
| return PagedResult(results=contents, next_page_token=next_page_token) | return PagedResult(results=contents, next_page_token=next_page_token) | ||||
| @timed | @timed | ||||
| @db_transaction(statement_timeout=500) | @db_transaction(statement_timeout=500) | ||||
| def content_get( | def content_get( | ||||
| self, contents: List[Sha1], db=None, cur=None | self, contents: List[bytes], algo: str = "sha1", db=None, cur=None | ||||
| ) -> List[Optional[Content]]: | ) -> List[Optional[Content]]: | ||||
| contents_by_sha1: Dict[Sha1, Optional[Content]] = {} | contents_by_hash: Dict[bytes, Optional[Content]] = {} | ||||
| for row in db.content_get_metadata_from_sha1s(contents, cur): | if algo not in DEFAULT_ALGORITHMS: | ||||
| raise StorageArgumentException( | |||||
| "algo should be one of {','.join(DEFAULT_ALGORITHMS)}" | |||||
| ) | |||||
| rows = db.content_get_metadata_from_hashes(contents, algo, cur) | |||||
| key = operator.attrgetter(algo) | |||||
| for row in rows: | |||||
| row_d = dict(zip(db.content_get_metadata_keys, row)) | row_d = dict(zip(db.content_get_metadata_keys, row)) | ||||
| content = Content(**row_d) | content = Content(**row_d) | ||||
| contents_by_sha1[content.sha1] = content | contents_by_hash[key(content)] = content | ||||
| return [contents_by_sha1.get(sha1) for sha1 in contents] | return [contents_by_hash.get(sha1) for sha1 in contents] | ||||
| @timed | @timed | ||||
| @db_transaction_generator() | @db_transaction_generator() | ||||
| def content_missing( | def content_missing( | ||||
| self, contents: List[Dict[str, Any]], key_hash: str = "sha1", db=None, cur=None | self, contents: List[Dict[str, Any]], key_hash: str = "sha1", db=None, cur=None | ||||
| ) -> Iterable[bytes]: | ) -> Iterable[bytes]: | ||||
| if key_hash not in DEFAULT_ALGORITHMS: | if key_hash not in DEFAULT_ALGORITHMS: | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| ▲ Show 20 Lines • Show All 1,155 Lines • Show Last 20 Lines | |||||