diff --git a/swh/vault/tests/conftest.py b/swh/vault/tests/conftest.py --- a/swh/vault/tests/conftest.py +++ b/swh/vault/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020 The Software Heritage developers +# Copyright (C) 2020-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -12,7 +12,6 @@ from pytest_postgresql import factories from swh.core.db.pytest_plugin import initialize_database_for_module, postgresql_fact -from swh.storage.postgresql.db import Db as StorageDb from swh.vault import get_vault from swh.vault.backend import VaultBackend @@ -35,13 +34,6 @@ yield pathlib.Path(tmpdir) -storage_postgresql_proc = factories.postgresql_proc( - dbname="storage", - load=[ - partial(initialize_database_for_module, "storage", StorageDb.current_version) - ], -) - vault_postgresql_proc = factories.postgresql_proc( dbname="vault", load=[ @@ -50,25 +42,15 @@ ) postgres_vault = postgresql_fact("vault_postgresql_proc") -postgres_storage = postgresql_fact( - "storage_postgresql_proc", - no_db_drop=True, # keep the db for performance reasons -) @pytest.fixture -def swh_vault_config(postgres_vault, postgres_storage, tmp_path) -> Dict[str, Any]: +def swh_vault_config(postgres_vault, tmp_path) -> Dict[str, Any]: tmp_path = str(tmp_path) return { "db": postgres_vault.dsn, "storage": { - "cls": "postgresql", - "db": postgres_storage.dsn, - "objstorage": { - "cls": "pathslicing", - "root": tmp_path, - "slicing": "0:1/1:5", - }, + "cls": "memory", }, "cache": { "cls": "pathslicing", diff --git a/swh/vault/tests/test_cookers.py b/swh/vault/tests/test_cookers.py --- a/swh/vault/tests/test_cookers.py +++ b/swh/vault/tests/test_cookers.py @@ -17,6 +17,7 @@ import unittest import unittest.mock +import attrs import dulwich.fastexport import dulwich.index import dulwich.objects @@ -31,6 +32,7 @@ Release, Revision, RevisionType, + SkippedContent, Snapshot, SnapshotBranch, TargetType, @@ -437,32 +439,25 @@ obj_id = hashutil.hash_to_bytes(obj_id_hex) swhid = CoreSWHID(object_type=ObjectType.DIRECTORY, object_id=obj_id) - # FIXME: storage.content_update() should be changed to allow things - # like that - with loader.storage.get_db().transaction() as cur: - cur.execute( - """update content set status = 'visible' - where sha1 = %s""", - (id_1,), - ) - cur.execute( - """update content set status = 'hidden' - where sha1 = %s""", - (id_2,), - ) - - cur.execute( - """ - insert into skipped_content - (sha1, sha1_git, sha256, blake2s256, length, reason) - select sha1, sha1_git, sha256, blake2s256, length, 'no reason' - from content - where sha1 = %s - """, - (id_3,), - ) - - cur.execute("delete from content where sha1 = %s", (id_3,)) + # alter the content of the storage + # 1/ make file 2 an hidden file object + loader.storage._allow_overwrite = True + cnt2 = attrs.evolve( + loader.storage.content_get([id_2])[0], status="hidden", data=file_2 + ) + loader.storage.content_add([cnt2]) + assert loader.storage.content_get([id_2])[0].status == "hidden" + + # 2/ make file 3 an skipped file object + cnt3 = loader.storage.content_get([id_3])[0].to_dict() + cnt3["status"] = "absent" + cnt3["reason"] = "no reason" + sk_cnt3 = SkippedContent.from_dict(cnt3) + loader.storage.skipped_content_add([sk_cnt3]) + # dirty dirty dirty... let's pretend it is the equivalent of writing sql + # queries in the postgresql backend + for hashkey in loader.storage._cql_runner._content_indexes: + loader.storage._cql_runner._content_indexes[hashkey].pop(cnt3[hashkey]) with cook_extract_directory(loader.storage, swhid) as p: assert (p / "file").read_bytes() == b"test1" @@ -828,32 +823,26 @@ loader = git_loader(str(rp)) loader.load() - # FIXME: storage.content_update() should be changed to allow things - # like that - with loader.storage.get_db().transaction() as cur: - cur.execute( - """update content set status = 'visible' - where sha1 = %s""", - (id_1,), - ) - cur.execute( - """update content set status = 'hidden' - where sha1 = %s""", - (id_2,), - ) - - cur.execute( - """ - insert into skipped_content - (sha1, sha1_git, sha256, blake2s256, length, reason) - select sha1, sha1_git, sha256, blake2s256, length, 'no reason' - from content - where sha1 = %s - """, - (id_3,), - ) + # alter the content of the storage + # 1/ make file 2 an hidden file object + loader.storage._allow_overwrite = True + cnt2 = attrs.evolve( + loader.storage.content_get([id_2])[0], status="hidden", data=file_2 + ) + loader.storage.content_add([cnt2]) + assert loader.storage.content_get([id_2])[0].status == "hidden" + + # 2/ make file 3 an skipped file object + cnt3 = loader.storage.content_get([id_3])[0].to_dict() + cnt3["status"] = "absent" + cnt3["reason"] = "no reason" + sk_cnt3 = SkippedContent.from_dict(cnt3) + loader.storage.skipped_content_add([sk_cnt3]) + # dirty dirty dirty... let's pretend it is the equivalent of writing sql + # queries in the postgresql backend + for hashkey in loader.storage._cql_runner._content_indexes: + loader.storage._cql_runner._content_indexes[hashkey].pop(cnt3[hashkey]) - cur.execute("delete from content where sha1 = %s", (id_3,)) return (loader, swhid) def check_revision_filtered_objects(self, ert, p, swhid): diff --git a/swh/vault/tests/test_git_bare_cooker.py b/swh/vault/tests/test_git_bare_cooker.py --- a/swh/vault/tests/test_git_bare_cooker.py +++ b/swh/vault/tests/test_git_bare_cooker.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021 The Software Heritage developers +# Copyright (C) 2021-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -11,6 +11,7 @@ import datetime import enum +from functools import partial import io import subprocess import tarfile @@ -21,7 +22,9 @@ import dulwich.repo import pytest from pytest import param +from pytest_postgresql import factories +from swh.core.db.pytest_plugin import initialize_database_for_module, postgresql_fact from swh.model.from_disk import DentryPerms from swh.model.model import ( Content, @@ -38,9 +41,27 @@ Timestamp, TimestampWithTimezone, ) +from swh.storage import get_storage +from swh.storage.postgresql.db import Db as StorageBackend from swh.vault.cookers.git_bare import GitBareCooker from swh.vault.in_memory_backend import InMemoryVaultBackend +storage_postgresql_proc = factories.postgresql_proc( + dbname="storage", + load=[ + partial( + initialize_database_for_module, "storage", StorageBackend.current_version + ) + ], +) + +storage_postgresql = postgresql_fact("storage_postgresql_proc", no_db_drop=True) + + +@pytest.fixture +def swh_storage(storage_postgresql): + return get_storage("local", db=storage_postgresql.dsn, objstorage={"cls": "memory"}) + class RootObjects(enum.Enum): REVISION = enum.auto()