swh_storage_backend_config = {'cls': 'local', 'db': 'postgresql://postgres@127.0.0.1:12458/tests', 'journal_writer': {'brokers': ['127.0.0.1:49109'], 'client_id': 'kafka_writer-1', 'cls': 'kafka', 'prefix': 'qtkovorxux-1'}, 'objstorage': {'args': {}, 'cls': 'memory'}}
kafka_prefix = 'qtkovorxux', kafka_consumer_group = 'test-consumer-qtkovorxux'
kafka_server = '127.0.0.1:49109'
@patch("swh.storage.backfill.RANGE_GENERATORS", RANGE_GENERATORS)
def test_backfiller(
swh_storage_backend_config,
kafka_prefix: str,
kafka_consumer_group: str,
kafka_server: str,
):
prefix1 = f"{kafka_prefix}-1"
prefix2 = f"{kafka_prefix}-2"
journal1 = {
"cls": "kafka",
"brokers": [kafka_server],
"client_id": "kafka_writer-1",
"prefix": prefix1,
}
swh_storage_backend_config["journal_writer"] = journal1
storage = get_storage(**swh_storage_backend_config)
# fill the storage and the journal (under prefix1)
for object_type, objects in TEST_OBJECTS.items():
method = getattr(storage, object_type + "_add")
method(objects)
# now apply the backfiller on the storage to fill the journal under prefix2
backfiller_config = {
"brokers": [kafka_server],
"client_id": "kafka_writer-2",
"prefix": prefix2,
"storage_dbconn": swh_storage_backend_config["db"],
}
# Backfilling
backfiller = JournalBackfiller(backfiller_config)
for object_type in TEST_OBJECTS:
> backfiller.run(object_type, None, None)
.tox/py3/lib/python3.7/site-packages/swh/storage/tests/test_backfill.py:217:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
.tox/py3/lib/python3.7/site-packages/swh/storage/backfill.py:512: in run
obj = object_converter_fn[object_type](obj_d)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cls = <class 'swh.model.model.Release'>
d = Release(name=b'v0.0.1', message=b'foo', target=b'\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\x04\...et=120, negative_utc=False), metadata=None, id=b'\x80Y\xdcN\x17\xfc\xd0\xe5\x1c\xa3\xbc\xd6\xb8\x0fEw\xd2\x81\xfd\x08')
@classmethod
def from_dict(cls, d):
> d = d.copy()
E AttributeError: 'Release' object has no attribute 'copy'
.tox/py3/lib/python3.7/site-packages/swh/model/model.py:416: AttributeError
TEST RESULT
TEST RESULT
- Run At
- Aug 12 2020, 11:01 AM