kafka_prefix = 'ezzguxilbv', kafka_consumer_group = 'test-consumer-ezzguxilbv'
kafka_server = '127.0.0.1:43655', privileged = False
@pytest.mark.parametrize("privileged", [True, False])
def test_storage_play_anonymized(
kafka_prefix: str, kafka_consumer_group: str, kafka_server: str, privileged: bool,
):
"""Optimal replayer scenario.
This:
- writes objects to the topic
- replayer consumes objects from the topic and replay them
This tests the behavior with both a privileged and non-privileged replayer
"""
writer_config = {
"cls": "kafka",
"brokers": [kafka_server],
"client_id": "kafka_writer",
"prefix": kafka_prefix,
"anonymize": True,
}
src_config: Dict[str, Any] = {"cls": "memory", "journal_writer": writer_config}
storage = get_storage(**src_config)
# Fill the src storage
nb_sent = 0
for obj_type, objs in TEST_OBJECTS.items():
if obj_type in ("origin_visit", "origin_visit_status"):
# these are unrelated with what we want to test here
continue
method = getattr(storage, obj_type + "_add")
> method(objs)
.tox/py3/lib/python3.7/site-packages/swh/storage/tests/test_replay.py:353:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <swh.storage.in_memory.InMemoryStorage object at 0x7fb77607f320>
directories = [Directory(entries=(), id=b'K\x82]\xc6B\xcbn\xb9\xa0`\xe5K\xf8\xd6\x92\x88\xfb\xeeI\x04'), Directory(entries=(Director...7<&z\xdcw\xc2\x90\x8ed\xe0\xc1\xcbjD1', perms=57344)), id=b'\x87\xb39\x10O}\xc2\xa8\x16=\xec\x98\x84E\xe3\x98y\x95T_')]
def directory_add(self, directories: List[Directory]) -> Dict[str, int]:
to_add = {d.id: d for d in directories}.values()
if not self._allow_overwrite:
# Filter out directories that are already inserted.
missing = self.directory_missing([dir_.id for dir_ in to_add])
directories = [dir_ for dir_ in directories if dir_.id in missing]
self.journal_writer.directory_add(directories)
for directory in directories:
# Add directory entries to the 'directory_entry' table
rows = [
DirectoryEntryRow(directory_id=directory.id, **entry.to_dict())
for entry in directory.entries
]
> if self._directory_entries_insert_algo == "one-by-one":
E AttributeError: 'InMemoryStorage' object has no attribute '_directory_entries_insert_algo'
.tox/py3/lib/python3.7/site-packages/swh/storage/cassandra/storage.py:451: AttributeError
TEST RESULT
TEST RESULT
- Run At
- Aug 26 2021, 11:17 AM