Page MenuHomeSoftware Heritage
Differential D8126 Diff 29358 swh/storage/tests/test_cassandra_migration.py

Changeset View

Standalone View

View Options

swh/storage/tests/test_cassandra_migration.py

Show First 20 Lines • Show All 117 Lines • ▼ Show 20 Lines) -> None:
for statement in CONTENT_INDEX_TEMPLATE.split("\n\n"): for statement in CONTENT_INDEX_TEMPLATE.split("\n\n"):
swh_storage._cql_runner._session.execute(statement.format(main_algo="byte_xor")) swh_storage._cql_runner._session.execute(statement.format(main_algo="byte_xor"))
# Should not affect the running code at all: # Should not affect the running code at all:
assert swh_storage.content_get([StorageData.content.sha1]) == [ assert swh_storage.content_get([StorageData.content.sha1]) == [
attr.evolve(StorageData.content, data=None) attr.evolve(StorageData.content, data=None)
] ]
with pytest.raises(StorageArgumentException): with pytest.raises(StorageArgumentException):
swh_storage.content_find({"byte_xor": content_xor_hash}) swh_storage.content_find({"byte_xor": content_xor_hash}) # type: ignore
# Then update the running code: # Then update the running code:
new_hash_algos = HASH_ALGORITHMS + ["byte_xor"] new_hash_algos = HASH_ALGORITHMS + ["byte_xor"]
mocker.patch("swh.storage.cassandra.storage.HASH_ALGORITHMS", new_hash_algos) mocker.patch("swh.storage.cassandra.storage.HASH_ALGORITHMS", new_hash_algos)
mocker.patch("swh.storage.cassandra.cql.HASH_ALGORITHMS", new_hash_algos) mocker.patch("swh.storage.cassandra.cql.HASH_ALGORITHMS", new_hash_algos)
mocker.patch("swh.model.model.DEFAULT_ALGORITHMS", new_hash_algos) mocker.patch("swh.model.model.DEFAULT_ALGORITHMS", new_hash_algos)
mocker.patch("swh.storage.cassandra.storage.Content", ContentWithXor) mocker.patch("swh.storage.cassandra.storage.Content", ContentWithXor)
mocker.patch("swh.storage.cassandra.storage.ContentRow", ContentRowWithXor) mocker.patch("swh.storage.cassandra.storage.ContentRow", ContentRowWithXor)
Show All 18 Lines) -> None:
swh_storage._set_cql_runner() swh_storage._set_cql_runner()
# Old algos still works, and return the new object type: # Old algos still works, and return the new object type:
assert swh_storage.content_get([StorageData.content.sha1]) == [ assert swh_storage.content_get([StorageData.content.sha1]) == [
attr.evolve(new_content, data=None, byte_xor=None) attr.evolve(new_content, data=None, byte_xor=None)
] ]
# The new algo does not work, we did not backfill it yet: # The new algo does not work, we did not backfill it yet:
assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [] assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [] # type: ignore
# A normal storage would not overwrite, because the object already exists, # A normal storage would not overwrite, because the object already exists,
# as it is not aware it is missing a field: # as it is not aware it is missing a field:
swh_storage.content_add([new_content, new_content2]) swh_storage.content_add([new_content, new_content2])
assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [] assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [] # type: ignore
# Backfill (in production this would be done with a replayer reading from # Backfill (in production this would be done with a replayer reading from
# the journal): # the journal):
overwriting_swh_storage = get_storage( overwriting_swh_storage = get_storage(
allow_overwrite=True, **swh_storage_backend_config allow_overwrite=True, **swh_storage_backend_config
) )
overwriting_swh_storage.content_add([new_content, new_content2]) overwriting_swh_storage.content_add([new_content, new_content2])
# Now, the object can be found: # Now, the object can be found:
assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [ assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [ # type: ignore
attr.evolve(new_content, data=None) attr.evolve(new_content, data=None)
] ]
############################## ##############################
# Test complex migrations # Test complex migrations
▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Lines) -> None:
for statement in CONTENT_INDEX_TEMPLATE.split("\n\n"): for statement in CONTENT_INDEX_TEMPLATE.split("\n\n"):
swh_storage._cql_runner._session.execute(statement.format(main_algo="byte_xor")) swh_storage._cql_runner._session.execute(statement.format(main_algo="byte_xor"))
# Should not affect the running code at all: # Should not affect the running code at all:
assert swh_storage.content_get([StorageData.content.sha1]) == [ assert swh_storage.content_get([StorageData.content.sha1]) == [
attr.evolve(StorageData.content, data=None) attr.evolve(StorageData.content, data=None)
] ]
with pytest.raises(StorageArgumentException): with pytest.raises(StorageArgumentException):
swh_storage.content_find({"byte_xor": content_xor_hash}) swh_storage.content_find({"byte_xor": content_xor_hash}) # type: ignore
# Then update the running code: # Then update the running code:
new_hash_algos = HASH_ALGORITHMS + ["byte_xor"] new_hash_algos = HASH_ALGORITHMS + ["byte_xor"]
mocker.patch("swh.storage.cassandra.storage.HASH_ALGORITHMS", new_hash_algos) mocker.patch("swh.storage.cassandra.storage.HASH_ALGORITHMS", new_hash_algos)
mocker.patch("swh.storage.cassandra.cql.HASH_ALGORITHMS", new_hash_algos) mocker.patch("swh.storage.cassandra.cql.HASH_ALGORITHMS", new_hash_algos)
mocker.patch("swh.model.model.DEFAULT_ALGORITHMS", new_hash_algos) mocker.patch("swh.model.model.DEFAULT_ALGORITHMS", new_hash_algos)
mocker.patch("swh.storage.cassandra.storage.Content", ContentWithXor) mocker.patch("swh.storage.cassandra.storage.Content", ContentWithXor)
mocker.patch("swh.storage.cassandra.storage.ContentRow", ContentRowWithXorPK) mocker.patch("swh.storage.cassandra.storage.ContentRow", ContentRowWithXorPK)
Show All 32 Lines) -> None:
# When the replayer gets close to the end of the logs, loaders are stopped # When the replayer gets close to the end of the logs, loaders are stopped
# to allow the replayer to catch up with the end of the log. # to allow the replayer to catch up with the end of the log.
# When it does, we can switch over to the new swh-storage's code. # When it does, we can switch over to the new swh-storage's code.
# Simulates a restart: # Simulates a restart:
swh_storage._set_cql_runner() swh_storage._set_cql_runner()
# Now, the object can be found with the new hash: # Now, the object can be found with the new hash:
assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [ assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [ # type: ignore
attr.evolve(new_content, data=None) attr.evolve(new_content, data=None)
] ]
# Remove the old table: # Remove the old table:
swh_storage._cql_runner._session.execute("DROP TABLE content") swh_storage._cql_runner._session.execute("DROP TABLE content")
# Object is still available, because we don't use it anymore # Object is still available, because we don't use it anymore
assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [ assert swh_storage.content_find({"byte_xor": content_xor_hash}) == [ # type: ignore
attr.evolve(new_content, data=None) attr.evolve(new_content, data=None)
] ]
# THE END. # THE END.
# Test teardown expects a table with this name to exist: # Test teardown expects a table with this name to exist:
swh_storage._cql_runner._session.execute( swh_storage._cql_runner._session.execute(
"CREATE TABLE content (foo blob PRIMARY KEY);" "CREATE TABLE content (foo blob PRIMARY KEY);"
) )
# Clean up this table, test teardown does not know about it: # Clean up this table, test teardown does not know about it:
swh_storage._cql_runner._session.execute("DROP TABLE content_v2;") swh_storage._cql_runner._session.execute("DROP TABLE content_v2;")
About · Developer information · Legal