Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_cassandra_migration.py
Show First 20 Lines • Show All 108 Lines • ▼ Show 20 Lines | ) -> None: | ||||
This is a simple migration, as it does not require an update to the primary key. | This is a simple migration, as it does not require an update to the primary key. | ||||
""" | """ | ||||
content_xor_hash = byte_xor_hash(StorageData.content.data) | content_xor_hash = byte_xor_hash(StorageData.content.data) | ||||
# First insert some existing data | # First insert some existing data | ||||
swh_storage.content_add([StorageData.content, StorageData.content2]) | swh_storage.content_add([StorageData.content, StorageData.content2]) | ||||
# Then update the schema | # Then update the schema | ||||
swh_storage._cql_runner._session.execute("ALTER TABLE content ADD byte_xor blob") | session = swh_storage._cql_runner._cluster.connect(swh_storage._cql_runner.keyspace) | ||||
session.execute("ALTER TABLE content ADD byte_xor blob") | |||||
for statement in CONTENT_INDEX_TEMPLATE.split("\n\n"): | for statement in CONTENT_INDEX_TEMPLATE.split("\n\n"): | ||||
swh_storage._cql_runner._session.execute(statement.format(main_algo="byte_xor")) | session.execute(statement.format(main_algo="byte_xor")) | ||||
# Should not affect the running code at all: | # Should not affect the running code at all: | ||||
assert swh_storage.content_get([StorageData.content.sha1]) == [ | assert swh_storage.content_get([StorageData.content.sha1]) == [ | ||||
attr.evolve(StorageData.content, data=None) | attr.evolve(StorageData.content, data=None) | ||||
] | ] | ||||
with pytest.raises(StorageArgumentException): | with pytest.raises(StorageArgumentException): | ||||
swh_storage.content_find({"byte_xor": content_xor_hash}) | swh_storage.content_find({"byte_xor": content_xor_hash}) | ||||
▲ Show 20 Lines • Show All 109 Lines • ▼ Show 20 Lines | |||||
def test_change_content_pk( | def test_change_content_pk( | ||||
swh_storage: CassandraStorage, swh_storage_backend_config, mocker # noqa | swh_storage: CassandraStorage, swh_storage_backend_config, mocker # noqa | ||||
) -> None: | ) -> None: | ||||
"""Adds a column to the 'content' table and a new matching index; | """Adds a column to the 'content' table and a new matching index; | ||||
and make this new column part of the primary key | and make this new column part of the primary key | ||||
This is a complex migration, as it requires copying the whole table | This is a complex migration, as it requires copying the whole table | ||||
""" | """ | ||||
content_xor_hash = byte_xor_hash(StorageData.content.data) | content_xor_hash = byte_xor_hash(StorageData.content.data) | ||||
session = swh_storage._cql_runner._cluster.connect(swh_storage._cql_runner.keyspace) | |||||
# First insert some existing data | # First insert some existing data | ||||
swh_storage.content_add([StorageData.content, StorageData.content2]) | swh_storage.content_add([StorageData.content, StorageData.content2]) | ||||
# Then add a new table and a new index | # Then add a new table and a new index | ||||
swh_storage._cql_runner._session.execute( | session.execute( | ||||
""" | """ | ||||
CREATE TABLE IF NOT EXISTS content_v2 ( | CREATE TABLE IF NOT EXISTS content_v2 ( | ||||
sha1 blob, | sha1 blob, | ||||
sha1_git blob, | sha1_git blob, | ||||
sha256 blob, | sha256 blob, | ||||
blake2s256 blob, | blake2s256 blob, | ||||
byte_xor blob, | byte_xor blob, | ||||
length bigint, | length bigint, | ||||
ctime timestamp, | ctime timestamp, | ||||
-- creation time, i.e. time of (first) injection into the storage | -- creation time, i.e. time of (first) injection into the storage | ||||
status ascii, | status ascii, | ||||
PRIMARY KEY ((sha1, sha1_git, sha256, blake2s256, byte_xor)) | PRIMARY KEY ((sha1, sha1_git, sha256, blake2s256, byte_xor)) | ||||
);""" | );""" | ||||
) | ) | ||||
for statement in CONTENT_INDEX_TEMPLATE.split("\n\n"): | for statement in CONTENT_INDEX_TEMPLATE.split("\n\n"): | ||||
swh_storage._cql_runner._session.execute(statement.format(main_algo="byte_xor")) | session.execute(statement.format(main_algo="byte_xor")) | ||||
# Should not affect the running code at all: | # Should not affect the running code at all: | ||||
assert swh_storage.content_get([StorageData.content.sha1]) == [ | assert swh_storage.content_get([StorageData.content.sha1]) == [ | ||||
attr.evolve(StorageData.content, data=None) | attr.evolve(StorageData.content, data=None) | ||||
] | ] | ||||
with pytest.raises(StorageArgumentException): | with pytest.raises(StorageArgumentException): | ||||
swh_storage.content_find({"byte_xor": content_xor_hash}) | swh_storage.content_find({"byte_xor": content_xor_hash}) | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | ) -> None: | ||||
swh_storage._set_cql_runner() | swh_storage._set_cql_runner() | ||||
# Now, the object can be found with the new hash: | # Now, the object can be found with the new hash: | ||||
assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [ | assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [ | ||||
attr.evolve(new_content, data=None) | attr.evolve(new_content, data=None) | ||||
] | ] | ||||
# Remove the old table: | # Remove the old table: | ||||
swh_storage._cql_runner._session.execute("DROP TABLE content") | session.execute("DROP TABLE content") | ||||
# Object is still available, because we don't use it anymore | # Object is still available, because we don't use it anymore | ||||
assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [ | assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [ | ||||
attr.evolve(new_content, data=None) | attr.evolve(new_content, data=None) | ||||
] | ] | ||||
# THE END. | # THE END. | ||||
# Test teardown expects a table with this name to exist: | # Test teardown expects a table with this name to exist: | ||||
swh_storage._cql_runner._session.execute( | session.execute("CREATE TABLE content (foo blob PRIMARY KEY);") | ||||
"CREATE TABLE content (foo blob PRIMARY KEY);" | |||||
) | |||||
# Clean up this table, test teardown does not know about it: | # Clean up this table, test teardown does not know about it: | ||||
swh_storage._cql_runner._session.execute("DROP TABLE content_v2;") | session.execute("DROP TABLE content_v2;") |