kafka_prefix = 'cbtzpcwypd', kafka_consumer_group = 'test-consumer-cbtzpcwypd'
kafka_server = '127.0.0.1:56057', privileged = False
@pytest.mark.parametrize("privileged", [True, False])
def test_storage_play_anonymized(
kafka_prefix: str, kafka_consumer_group: str, kafka_server: str, privileged: bool,
):
"""Optimal replayer scenario.
This:
- writes objects to the topic
- replayer consumes objects from the topic and replay them
This tests the behavior with both a privileged and non-privileged replayer
"""
writer_config = {
"cls": "kafka",
"brokers": [kafka_server],
"client_id": "kafka_writer",
"prefix": kafka_prefix,
"anonymize": True,
}
src_config: Dict[str, Any] = {"cls": "memory", "journal_writer": writer_config}
storage = get_storage(**src_config)
# Fill the src storage
nb_sent = 0
for obj_type, objs in TEST_OBJECTS.items():
if obj_type in ("origin_visit", "origin_visit_status"):
# these are unrelated with what we want to test here
continue
method = getattr(storage, obj_type + "_add")
> method(objs)
.tox/py3/lib/python3.7/site-packages/swh/storage/tests/test_replay.py:353:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
.tox/py3/lib/python3.7/site-packages/swh/storage/metrics.py:24: in d
return f(*a, **kw)
.tox/py3/lib/python3.7/site-packages/swh/storage/metrics.py:77: in d
r = f(*a, **kw)
.tox/py3/lib/python3.7/site-packages/swh/storage/cassandra/storage.py:265: in content_add
return self._content_add(list(contents), with_data=True)
.tox/py3/lib/python3.7/site-packages/swh/storage/cassandra/storage.py:221: in _content_add
for row in rows:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <swh.storage.in_memory.InMemoryStorage object at 0x7f003b90b5f8>
algo = 'sha1_git'
hashes = [b'\x86\xbck7~\x9d%\xf9\xd2gw\xa4\xa2\x8d\x08\xe6>|Wy']
def _content_get_from_hashes(self, algo, hashes: List[bytes]) -> Iterable:
"""From the name of a hash algorithm and a value of that hash,
looks up the "hash -> token" secondary table (content_by_{algo})
to get tokens.
Then, looks up the main table (content) to get all contents with
that token, and filters out contents whose hash doesn't match."""
found_tokens = list(
self._cql_runner.content_get_tokens_from_single_algo(algo, hashes)
)
assert all(isinstance(token, int) for token in found_tokens)
# Query the main table ('content').
> rows = self._cql_runner.content_get_from_tokens(found_tokens)
E AttributeError: 'InMemoryCqlRunner' object has no attribute 'content_get_from_tokens'
.tox/py3/lib/python3.7/site-packages/swh/storage/cassandra/storage.py:172: AttributeError
TEST RESULT
TEST RESULT
- Run At
- Sep 15 2021, 3:20 PM