Differential D5582 Diff 20236 swh/storage/tests/test_cassandra.py

Changeset View

Standalone View

swh/storage/tests/test_cassandra.py

# Copyright (C) 2018-2021 The Software Heritage developers		# Copyright (C) 2018-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import datetime		import datetime
		import itertools
import os		import os
import signal		import signal
import socket		import socket
import subprocess		import subprocess
import time		import time
from typing import Dict		from typing import Any, Dict

import attr		import attr
import pytest		import pytest

from swh.core.api.classes import stream_results		from swh.core.api.classes import stream_results
		from swh.model.model import Directory, DirectoryEntry, Snapshot
from swh.storage import get_storage		from swh.storage import get_storage
from swh.storage.cassandra import create_keyspace		from swh.storage.cassandra import create_keyspace
from swh.storage.cassandra.model import ContentRow, ExtIDRow		from swh.storage.cassandra.model import ContentRow, ExtIDRow
from swh.storage.cassandra.schema import HASH_ALGORITHMS, TABLES		from swh.storage.cassandra.schema import HASH_ALGORITHMS, TABLES
		from swh.storage.tests.storage_data import StorageData
from swh.storage.tests.storage_tests import (		from swh.storage.tests.storage_tests import (
TestStorageGeneratedData as _TestStorageGeneratedData,		TestStorageGeneratedData as _TestStorageGeneratedData,
)		)
from swh.storage.tests.storage_tests import TestStorage as _TestStorage		from swh.storage.tests.storage_tests import TestStorage as _TestStorage
from swh.storage.utils import now		from swh.storage.utils import now, remove_keys

CONFIG_TEMPLATE = """		CONFIG_TEMPLATE = """
data_file_directories:		data_file_directories:
- {data_dir}/data		- {data_dir}/data
commitlog_directory: {data_dir}/commitlog		commitlog_directory: {data_dir}/commitlog
hints_directory: {data_dir}/hints		hints_directory: {data_dir}/hints
saved_caches_directory: {data_dir}/saved_caches		saved_caches_directory: {data_dir}/saved_caches

▲ Show 20 Lines • Show All 408 Lines • ▼ Show 20 Lines	class TestCassandraStorageGeneratedData(_TestStorageGeneratedData):

@pytest.mark.skip("Not supported by Cassandra")		@pytest.mark.skip("Not supported by Cassandra")
def test_origin_count_with_visit_with_visits_and_snapshot(self):		def test_origin_count_with_visit_with_visits_and_snapshot(self):
pass		pass

@pytest.mark.skip("Not supported by Cassandra")		@pytest.mark.skip("Not supported by Cassandra")
def test_origin_count_with_visit_with_visits_no_snapshot(self):		def test_origin_count_with_visit_with_visits_no_snapshot(self):
pass		pass


		@pytest.mark.parametrize(
		"allow_overwrite,object_type",
		itertools.product(
		[False, True],
		# Note the absence of "content", it's tested above.
		["directory", "revision", "release", "snapshot", "origin", "extid"],
		),
		)
		def test_allow_overwrite(
		allow_overwrite: bool, object_type: str, swh_storage_backend_config
		):
		if object_type in ("origin", "extid"):
		pytest.skip(
		f"test_disallow_overwrite not implemented for {object_type} objects, "
		f"because all their columns are in the primary key."
		)
		swh_storage = get_storage(
		allow_overwrite=allow_overwrite, **swh_storage_backend_config
		)

		# directory_ls joins with content and directory table, and needs those to return
		# non-None entries:
		if object_type == "directory":
		swh_storage.directory_add([StorageData.directory5])
		swh_storage.content_add([StorageData.content, StorageData.content2])

		obj1: Any
		obj2: Any

		# Get two test objects
		if object_type == "directory":
		(obj1, obj2, *_) = StorageData.directories
		elif object_type == "snapshot":
		# StorageData.snapshots[1] is the empty snapshot, which is the corner case
		# that makes this test succeed for the wrong reasons
		obj1 = StorageData.snapshot
		obj2 = StorageData.complete_snapshot
		else:
		(obj1, obj2, *_) = getattr(StorageData, (object_type + "s"))

		# Let's make both objects have the same hash, but different content
		obj1 = attr.evolve(obj1, id=obj2.id)

		# Get the methods used to add and get these objects
		add = getattr(swh_storage, object_type + "_add")
		if object_type == "directory":

		def get(ids):
		return [
		Directory(
		id=ids[0],
		entries=tuple(
		map(
		lambda entry: DirectoryEntry(
		name=entry["name"],
		type=entry["type"],
		target=entry["sha1_git"],
		perms=entry["perms"],
		),
		swh_storage.directory_ls(ids[0]),
		)
		),
		)
		]

		elif object_type == "snapshot":

		def get(ids):
		return [
		Snapshot.from_dict(
		remove_keys(swh_storage.snapshot_get(ids[0]), ("next_branch",))
		)
		]

		else:
		get = getattr(swh_storage, object_type + "_get")

		# Add the first object
		add([obj1])

		# It should be returned as-is
		assert get([obj1.id]) == [obj1]

		# Add the second object
		add([obj2])

		if allow_overwrite:
		# obj1 was overwritten by obj2
		expected = obj2
		else:
		# obj2 was not written, because obj1 already exists and has the same hash
		expected = obj1

		if allow_overwrite and object_type in ("directory", "snapshot"):
		# TODO
		pytest.xfail(
		"directory entries and snapshot branches are concatenated "
		"instead of being replaced"
		)
		assert get([obj1.id]) == [expected]