Page MenuHomeSoftware Heritage
Differential D7929

use an in-memory storage for tests
ClosedPublic
Actions

Authored by douardda on May 31 2022, 6:39 PM.

Details

Reviewers
vlorentz
Group Reviewers
Reviewers
Maniphest Tasks
T4289: Replace usage of swh.core's postgresql_fact by stock pytest_postgresql's factory function
Commits
rDVAUb7b7927ec97b: use an in-memory storage for test_cookers and test_to_disk
Summary

to reduce test execution time a bit.

Related to T4289.

Diff Detail

Event Timeline

douardda created this revision.May 31 2022, 6:39 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptMay 31 2022, 6:39 PM
douardda added a child revision: D7930: tests: use stock pytest_postgresql factory function.May 31 2022, 6:39 PM
swh-public-ci added a comment.May 31 2022, 6:42 PM

Build has FAILED

Patch application report for D7929 (id=28565)

Rebasing onto 922a1ea0aa...

Current branch diff-target is up to date.
Changes applied before test
commit 83b4cad7af66c1076fc34a804a158b6c04e3d2d1
Author: David Douard <david.douard@sdfa3.org>
Date:   Tue May 31 18:24:03 2022 +0200

    use an in-memory storage for tests
    
    to reduce test execution time a bit.

Link to build: https://jenkins.softwareheritage.org/job/DVAU/job/tests-on-diff/232/
See console output for more information: https://jenkins.softwareheritage.org/job/DVAU/job/tests-on-diff/232/console

Harbormaster returned this revision to the author for changes because remote builds failed.May 31 2022, 6:42 PM
Harbormaster failed remote builds in B29650: Diff 28565!
douardda updated this revision to Diff 28571.May 31 2022, 7:27 PM

keep test_git_bare_cooker using a postgresql storage

this one really needs it.

swh-public-ci added a comment.May 31 2022, 7:31 PM

Build is green

Patch application report for D7929 (id=28571)

Rebasing onto 922a1ea0aa...

Current branch diff-target is up to date.
Changes applied before test
commit b7b7927ec97b61c4943a6b6f1eb17a72ca19dae1
Author: David Douard <david.douard@sdfa3.org>
Date:   Tue May 31 18:24:03 2022 +0200

    use an in-memory storage for test_cookers and test_to_disk
    
    to reduce test execution time a bit.
    Keep test_git_bare_cooker using a postgresql backend, since it depends
    on it (support for displayname).

See https://jenkins.softwareheritage.org/job/DVAU/job/tests-on-diff/234/ for more details.

Harbormaster completed remote builds in B29656: Diff 28571.May 31 2022, 7:31 PM
douardda requested review of this revision.May 31 2022, 7:31 PM
vlorentz accepted this revision.May 31 2022, 7:32 PM
vlorentz added a subscriber: vlorentz.

nice

This revision is now accepted and ready to land.May 31 2022, 7:32 PM
Closed by commit rDVAUb7b7927ec97b: use an in-memory storage for test_cookers and test_to_disk (authored by douardda). · Explain WhyJun 1 2022, 9:37 AM
This revision was automatically updated to reflect the committed changes.
douardda added a commit: rDVAUb7b7927ec97b: use an in-memory storage for test_cookers and test_to_disk.

Revision Contents
Changeset List

PathSizeCoverage (All)Coverage (Touched)
swh/
vault/
tests/
24 lines82%100%
91 lines99%100%
23 lines98%100%

Diff 28571

View Options

swh/vault/tests/conftest.py

# Copyright (C) 2020 The Software Heritage developers # Copyright (C) 2020-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
from functools import partial from functools import partial
import os import os
from typing import Any, Dict from typing import Any, Dict
import pkg_resources.extern.packaging.version import pkg_resources.extern.packaging.version
import pytest import pytest
from pytest_postgresql import factories from pytest_postgresql import factories
from swh.core.db.pytest_plugin import initialize_database_for_module, postgresql_fact from swh.core.db.pytest_plugin import initialize_database_for_module, postgresql_fact
from swh.storage.postgresql.db import Db as StorageDb
from swh.vault import get_vault from swh.vault import get_vault
from swh.vault.backend import VaultBackend from swh.vault.backend import VaultBackend
os.environ["LC_ALL"] = "C.UTF-8" os.environ["LC_ALL"] = "C.UTF-8"
# needed for directory tests on git-cloned repositories # needed for directory tests on git-cloned repositories
# 022 is usually the default value, but some environments (eg. Debian builds) have # 022 is usually the default value, but some environments (eg. Debian builds) have
# a different one. # a different one.
os.umask(0o022) os.umask(0o022)
pytest_v = pkg_resources.get_distribution("pytest").parsed_version pytest_v = pkg_resources.get_distribution("pytest").parsed_version
if pytest_v < pkg_resources.extern.packaging.version.parse("3.9"): if pytest_v < pkg_resources.extern.packaging.version.parse("3.9"):
@pytest.fixture @pytest.fixture
def tmp_path(): def tmp_path():
import pathlib import pathlib
import tempfile import tempfile
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir:
yield pathlib.Path(tmpdir) yield pathlib.Path(tmpdir)
storage_postgresql_proc = factories.postgresql_proc(
dbname="storage",
load=[
partial(initialize_database_for_module, "storage", StorageDb.current_version)
],
)
vault_postgresql_proc = factories.postgresql_proc( vault_postgresql_proc = factories.postgresql_proc(
dbname="vault", dbname="vault",
load=[ load=[
partial(initialize_database_for_module, "vault", VaultBackend.current_version) partial(initialize_database_for_module, "vault", VaultBackend.current_version)
], ],
) )
postgres_vault = postgresql_fact("vault_postgresql_proc") postgres_vault = postgresql_fact("vault_postgresql_proc")
postgres_storage = postgresql_fact(
"storage_postgresql_proc",
no_db_drop=True, # keep the db for performance reasons
)
@pytest.fixture @pytest.fixture
def swh_vault_config(postgres_vault, postgres_storage, tmp_path) -> Dict[str, Any]: def swh_vault_config(postgres_vault, tmp_path) -> Dict[str, Any]:
tmp_path = str(tmp_path) tmp_path = str(tmp_path)
return { return {
"db": postgres_vault.dsn, "db": postgres_vault.dsn,
"storage": { "storage": {
"cls": "postgresql", "cls": "memory",
"db": postgres_storage.dsn,
"objstorage": {
"cls": "pathslicing",
"root": tmp_path,
"slicing": "0:1/1:5",
},
}, },
"cache": { "cache": {
"cls": "pathslicing", "cls": "pathslicing",
"root": tmp_path, "root": tmp_path,
"slicing": "0:1/1:5", "slicing": "0:1/1:5",
"allow_delete": True, "allow_delete": True,
}, },
"scheduler": { "scheduler": {
Show All 14 Lines
View Options

swh/vault/tests/test_cookers.py

Show All 11 Lines
import pathlib import pathlib
import shutil import shutil
import subprocess import subprocess
import tarfile import tarfile
import tempfile import tempfile
import unittest import unittest
import unittest.mock import unittest.mock
import attrs
import dulwich.fastexport import dulwich.fastexport
import dulwich.index import dulwich.index
import dulwich.objects import dulwich.objects
import dulwich.porcelain import dulwich.porcelain
import dulwich.repo import dulwich.repo
import pytest import pytest
from swh.loader.git.from_disk import GitLoaderFromDisk from swh.loader.git.from_disk import GitLoaderFromDisk
from swh.model import from_disk, hashutil from swh.model import from_disk, hashutil
from swh.model.model import ( from swh.model.model import (
Person, Person,
Release, Release,
Revision, Revision,
RevisionType, RevisionType,
SkippedContent,
Snapshot, Snapshot,
SnapshotBranch, SnapshotBranch,
TargetType, TargetType,
Timestamp, Timestamp,
TimestampWithTimezone, TimestampWithTimezone,
) )
from swh.model.model import Content, Directory, DirectoryEntry from swh.model.model import Content, Directory, DirectoryEntry
from swh.model.model import ObjectType as ModelObjectType from swh.model.model import ObjectType as ModelObjectType
▲ Show 20 Lines • Show All 390 Lines • ▼ Show 20 Lines def test_directory_filtered_objects(self, git_loader, cook_extract_directory):
c = repo.commit() c = repo.commit()
loader = git_loader(str(rp)) loader = git_loader(str(rp))
loader.load() loader.load()
obj_id_hex = repo.repo[c].tree.decode() obj_id_hex = repo.repo[c].tree.decode()
obj_id = hashutil.hash_to_bytes(obj_id_hex) obj_id = hashutil.hash_to_bytes(obj_id_hex)
swhid = CoreSWHID(object_type=ObjectType.DIRECTORY, object_id=obj_id) swhid = CoreSWHID(object_type=ObjectType.DIRECTORY, object_id=obj_id)
# FIXME: storage.content_update() should be changed to allow things # alter the content of the storage
# like that # 1/ make file 2 an hidden file object
with loader.storage.get_db().transaction() as cur: loader.storage._allow_overwrite = True
cur.execute( cnt2 = attrs.evolve(
"""update content set status = 'visible' loader.storage.content_get([id_2])[0], status="hidden", data=file_2
where sha1 = %s""", )
(id_1,), loader.storage.content_add([cnt2])
) assert loader.storage.content_get([id_2])[0].status == "hidden"
cur.execute(
"""update content set status = 'hidden' # 2/ make file 3 an skipped file object
where sha1 = %s""", cnt3 = loader.storage.content_get([id_3])[0].to_dict()
(id_2,), cnt3["status"] = "absent"
) cnt3["reason"] = "no reason"
sk_cnt3 = SkippedContent.from_dict(cnt3)
cur.execute( loader.storage.skipped_content_add([sk_cnt3])
""" # dirty dirty dirty... let's pretend it is the equivalent of writing sql
insert into skipped_content # queries in the postgresql backend
(sha1, sha1_git, sha256, blake2s256, length, reason) for hashkey in loader.storage._cql_runner._content_indexes:
select sha1, sha1_git, sha256, blake2s256, length, 'no reason' loader.storage._cql_runner._content_indexes[hashkey].pop(cnt3[hashkey])
from content
where sha1 = %s
""",
(id_3,),
)
cur.execute("delete from content where sha1 = %s", (id_3,))
with cook_extract_directory(loader.storage, swhid) as p: with cook_extract_directory(loader.storage, swhid) as p:
assert (p / "file").read_bytes() == b"test1" assert (p / "file").read_bytes() == b"test1"
assert (p / "hidden_file").read_bytes() == HIDDEN_MESSAGE assert (p / "hidden_file").read_bytes() == HIDDEN_MESSAGE
assert (p / "absent_file").read_bytes() == SKIPPED_MESSAGE assert (p / "absent_file").read_bytes() == SKIPPED_MESSAGE
def test_directory_bogus_perms(self, git_loader, cook_extract_directory): def test_directory_bogus_perms(self, git_loader, cook_extract_directory):
# Some early git repositories have 664/775 permissions... let's check # Some early git repositories have 664/775 permissions... let's check
▲ Show 20 Lines • Show All 349 Lines • ▼ Show 20 Lines def load_repo_filtered_objects(self, git_loader):
repo.commit() repo.commit()
obj_id_hex = repo.repo.refs[b"HEAD"].decode() obj_id_hex = repo.repo.refs[b"HEAD"].decode()
obj_id = hashutil.hash_to_bytes(obj_id_hex) obj_id = hashutil.hash_to_bytes(obj_id_hex)
swhid = CoreSWHID(object_type=ObjectType.REVISION, object_id=obj_id) swhid = CoreSWHID(object_type=ObjectType.REVISION, object_id=obj_id)
loader = git_loader(str(rp)) loader = git_loader(str(rp))
loader.load() loader.load()
# FIXME: storage.content_update() should be changed to allow things # alter the content of the storage
# like that # 1/ make file 2 an hidden file object
with loader.storage.get_db().transaction() as cur: loader.storage._allow_overwrite = True
cur.execute( cnt2 = attrs.evolve(
"""update content set status = 'visible' loader.storage.content_get([id_2])[0], status="hidden", data=file_2
where sha1 = %s""", )
(id_1,), loader.storage.content_add([cnt2])
) assert loader.storage.content_get([id_2])[0].status == "hidden"
cur.execute(
"""update content set status = 'hidden' # 2/ make file 3 an skipped file object
where sha1 = %s""", cnt3 = loader.storage.content_get([id_3])[0].to_dict()
(id_2,), cnt3["status"] = "absent"
) cnt3["reason"] = "no reason"
sk_cnt3 = SkippedContent.from_dict(cnt3)
cur.execute( loader.storage.skipped_content_add([sk_cnt3])
""" # dirty dirty dirty... let's pretend it is the equivalent of writing sql
insert into skipped_content # queries in the postgresql backend
(sha1, sha1_git, sha256, blake2s256, length, reason) for hashkey in loader.storage._cql_runner._content_indexes:
select sha1, sha1_git, sha256, blake2s256, length, 'no reason' loader.storage._cql_runner._content_indexes[hashkey].pop(cnt3[hashkey])
from content
where sha1 = %s
""",
(id_3,),
)
cur.execute("delete from content where sha1 = %s", (id_3,))
return (loader, swhid) return (loader, swhid)
def check_revision_filtered_objects(self, ert, p, swhid): def check_revision_filtered_objects(self, ert, p, swhid):
ert.checkout(b"HEAD") ert.checkout(b"HEAD")
assert (p / "file").read_bytes() == b"test1" assert (p / "file").read_bytes() == b"test1"
assert (p / "hidden_file").read_bytes() == HIDDEN_MESSAGE assert (p / "hidden_file").read_bytes() == HIDDEN_MESSAGE
assert (p / "absent_file").read_bytes() == SKIPPED_MESSAGE assert (p / "absent_file").read_bytes() == SKIPPED_MESSAGE
▲ Show 20 Lines • Show All 347 Lines • Show Last 20 Lines
View Options

swh/vault/tests/test_git_bare_cooker.py

# Copyright (C) 2021 The Software Heritage developers # Copyright (C) 2021-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
""" """
This module contains additional tests for the bare cooker. This module contains additional tests for the bare cooker.
Generic cooker tests (eg. without swh-graph) in test_cookers.py also Generic cooker tests (eg. without swh-graph) in test_cookers.py also
run on the bare cooker. run on the bare cooker.
""" """
import datetime import datetime
import enum import enum
from functools import partial
import io import io
import subprocess import subprocess
import tarfile import tarfile
import tempfile import tempfile
import unittest.mock import unittest.mock
import attr import attr
import dulwich.repo import dulwich.repo
import pytest import pytest
from pytest import param from pytest import param
from pytest_postgresql import factories
from swh.core.db.pytest_plugin import initialize_database_for_module, postgresql_fact
from swh.model.from_disk import DentryPerms from swh.model.from_disk import DentryPerms
from swh.model.model import ( from swh.model.model import (
Content, Content,
Directory, Directory,
DirectoryEntry, DirectoryEntry,
ObjectType, ObjectType,
Person, Person,
Release, Release,
Revision, Revision,
RevisionType, RevisionType,
Snapshot, Snapshot,
SnapshotBranch, SnapshotBranch,
TargetType, TargetType,
Timestamp, Timestamp,
TimestampWithTimezone, TimestampWithTimezone,
) )
from swh.storage import get_storage
from swh.storage.postgresql.db import Db as StorageBackend
from swh.vault.cookers.git_bare import GitBareCooker from swh.vault.cookers.git_bare import GitBareCooker
from swh.vault.in_memory_backend import InMemoryVaultBackend from swh.vault.in_memory_backend import InMemoryVaultBackend
storage_postgresql_proc = factories.postgresql_proc(
dbname="storage",
load=[
partial(
initialize_database_for_module, "storage", StorageBackend.current_version
)
],
)
storage_postgresql = postgresql_fact("storage_postgresql_proc", no_db_drop=True)
@pytest.fixture
def swh_storage(storage_postgresql):
return get_storage("local", db=storage_postgresql.dsn, objstorage={"cls": "memory"})
class RootObjects(enum.Enum): class RootObjects(enum.Enum):
REVISION = enum.auto() REVISION = enum.auto()
SNAPSHOT = enum.auto() SNAPSHOT = enum.auto()
RELEASE = enum.auto() RELEASE = enum.auto()
WEIRD_RELEASE = enum.auto() # has a : in the name + points to another release WEIRD_RELEASE = enum.auto() # has a : in the name + points to another release
▲ Show 20 Lines • Show All 630 Lines • Show Last 20 Lines
About · Developer information · Legal