Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_cassandra.py
# Copyright (C) 2018-2021 The Software Heritage developers | # Copyright (C) 2018-2021 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import datetime | import datetime | ||||
import itertools | |||||
import os | import os | ||||
import signal | import signal | ||||
import socket | import socket | ||||
import subprocess | import subprocess | ||||
import time | import time | ||||
from typing import Dict | from typing import Any, Dict | ||||
import attr | import attr | ||||
import pytest | import pytest | ||||
from swh.core.api.classes import stream_results | from swh.core.api.classes import stream_results | ||||
from swh.model.model import Directory, DirectoryEntry, Snapshot | |||||
from swh.storage import get_storage | from swh.storage import get_storage | ||||
from swh.storage.cassandra import create_keyspace | from swh.storage.cassandra import create_keyspace | ||||
from swh.storage.cassandra.model import ContentRow, ExtIDRow | from swh.storage.cassandra.model import ContentRow, ExtIDRow | ||||
from swh.storage.cassandra.schema import HASH_ALGORITHMS, TABLES | from swh.storage.cassandra.schema import HASH_ALGORITHMS, TABLES | ||||
from swh.storage.tests.storage_data import StorageData | |||||
from swh.storage.tests.storage_tests import ( | from swh.storage.tests.storage_tests import ( | ||||
TestStorageGeneratedData as _TestStorageGeneratedData, | TestStorageGeneratedData as _TestStorageGeneratedData, | ||||
) | ) | ||||
from swh.storage.tests.storage_tests import TestStorage as _TestStorage | from swh.storage.tests.storage_tests import TestStorage as _TestStorage | ||||
from swh.storage.utils import now | from swh.storage.utils import now, remove_keys | ||||
CONFIG_TEMPLATE = """ | CONFIG_TEMPLATE = """ | ||||
data_file_directories: | data_file_directories: | ||||
- {data_dir}/data | - {data_dir}/data | ||||
commitlog_directory: {data_dir}/commitlog | commitlog_directory: {data_dir}/commitlog | ||||
hints_directory: {data_dir}/hints | hints_directory: {data_dir}/hints | ||||
saved_caches_directory: {data_dir}/saved_caches | saved_caches_directory: {data_dir}/saved_caches | ||||
▲ Show 20 Lines • Show All 408 Lines • ▼ Show 20 Lines | class TestCassandraStorageGeneratedData(_TestStorageGeneratedData): | ||||
@pytest.mark.skip("Not supported by Cassandra") | @pytest.mark.skip("Not supported by Cassandra") | ||||
def test_origin_count_with_visit_with_visits_and_snapshot(self): | def test_origin_count_with_visit_with_visits_and_snapshot(self): | ||||
pass | pass | ||||
@pytest.mark.skip("Not supported by Cassandra") | @pytest.mark.skip("Not supported by Cassandra") | ||||
def test_origin_count_with_visit_with_visits_no_snapshot(self): | def test_origin_count_with_visit_with_visits_no_snapshot(self): | ||||
pass | pass | ||||
@pytest.mark.parametrize( | |||||
"allow_overwrite,object_type", | |||||
itertools.product( | |||||
[False, True], | |||||
# Note the absence of "content", it's tested above. | |||||
["directory", "revision", "release", "snapshot", "origin", "extid"], | |||||
), | |||||
) | |||||
def test_allow_overwrite( | |||||
allow_overwrite: bool, object_type: str, swh_storage_backend_config | |||||
): | |||||
if object_type in ("origin", "extid"): | |||||
pytest.skip( | |||||
f"test_disallow_overwrite not implemented for {object_type} objects, " | |||||
f"because all their columns are in the primary key." | |||||
) | |||||
swh_storage = get_storage( | |||||
allow_overwrite=allow_overwrite, **swh_storage_backend_config | |||||
) | |||||
# directory_ls joins with content and directory table, and needs those to return | |||||
# non-None entries: | |||||
if object_type == "directory": | |||||
swh_storage.directory_add([StorageData.directory5]) | |||||
swh_storage.content_add([StorageData.content, StorageData.content2]) | |||||
obj1: Any | |||||
obj2: Any | |||||
# Get two test objects | |||||
if object_type == "directory": | |||||
(obj1, obj2, *_) = StorageData.directories | |||||
elif object_type == "snapshot": | |||||
# StorageData.snapshots[1] is the empty snapshot, which is the corner case | |||||
# that makes this test succeed for the wrong reasons | |||||
obj1 = StorageData.snapshot | |||||
obj2 = StorageData.complete_snapshot | |||||
else: | |||||
(obj1, obj2, *_) = getattr(StorageData, (object_type + "s")) | |||||
# Let's make both objects have the same hash, but different content | |||||
obj1 = attr.evolve(obj1, id=obj2.id) | |||||
# Get the methods used to add and get these objects | |||||
add = getattr(swh_storage, object_type + "_add") | |||||
if object_type == "directory": | |||||
def get(ids): | |||||
return [ | |||||
Directory( | |||||
id=ids[0], | |||||
entries=tuple( | |||||
map( | |||||
lambda entry: DirectoryEntry( | |||||
name=entry["name"], | |||||
type=entry["type"], | |||||
target=entry["sha1_git"], | |||||
perms=entry["perms"], | |||||
), | |||||
swh_storage.directory_ls(ids[0]), | |||||
) | |||||
), | |||||
) | |||||
] | |||||
elif object_type == "snapshot": | |||||
def get(ids): | |||||
return [ | |||||
Snapshot.from_dict( | |||||
remove_keys(swh_storage.snapshot_get(ids[0]), ("next_branch",)) | |||||
) | |||||
] | |||||
else: | |||||
get = getattr(swh_storage, object_type + "_get") | |||||
# Add the first object | |||||
add([obj1]) | |||||
# It should be returned as-is | |||||
assert get([obj1.id]) == [obj1] | |||||
# Add the second object | |||||
add([obj2]) | |||||
if allow_overwrite: | |||||
# obj1 was overwritten by obj2 | |||||
expected = obj2 | |||||
else: | |||||
# obj2 was not written, because obj1 already exists and has the same hash | |||||
expected = obj1 | |||||
if allow_overwrite and object_type in ("directory", "snapshot"): | |||||
# TODO | |||||
pytest.xfail( | |||||
"directory entries and snapshot branches are concatenated " | |||||
"instead of being replaced" | |||||
) | |||||
assert get([obj1.id]) == [expected] |