Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_buffer.py
Show All 21 Lines | missing_contents = storage.content_missing( | ||||
[contents[0], contents[1]]) | [contents[0], contents[1]]) | ||||
assert set(missing_contents) == set( | assert set(missing_contents) == set( | ||||
[contents[0]['sha1'], contents[1]['sha1']]) | [contents[0]['sha1'], contents[1]['sha1']]) | ||||
s = storage.flush() | s = storage.flush() | ||||
assert s == { | assert s == { | ||||
'content:add': 1 + 1, | 'content:add': 1 + 1, | ||||
'content:add:bytes': contents[0]['length'] + contents[1]['length'], | 'content:add:bytes': contents[0]['length'] + contents[1]['length'], | ||||
'skipped_content:add': 0 | |||||
} | } | ||||
missing_contents = storage.content_missing( | missing_contents = storage.content_missing( | ||||
[contents[0], contents[1]]) | [contents[0], contents[1]]) | ||||
assert list(missing_contents) == [] | assert list(missing_contents) == [] | ||||
def test_buffering_proxy_storage_content_threshold_nb_hit(sample_data): | def test_buffering_proxy_storage_content_threshold_nb_hit(sample_data): | ||||
contents = sample_data['content'] | contents = sample_data['content'] | ||||
storage = BufferingProxyStorage( | storage = BufferingProxyStorage( | ||||
storage={'cls': 'memory'}, | storage={'cls': 'memory'}, | ||||
min_batch_size={ | min_batch_size={ | ||||
'content': 1, | 'content': 1, | ||||
} | } | ||||
) | ) | ||||
s = storage.content_add([contents[0]]) | s = storage.content_add([contents[0]]) | ||||
assert s == { | assert s == { | ||||
'content:add': 1, | 'content:add': 1, | ||||
'content:add:bytes': contents[0]['length'], | 'content:add:bytes': contents[0]['length'], | ||||
'skipped_content:add': 0 | |||||
} | } | ||||
missing_contents = storage.content_missing([contents[0]]) | missing_contents = storage.content_missing([contents[0]]) | ||||
assert list(missing_contents) == [] | assert list(missing_contents) == [] | ||||
s = storage.flush() | s = storage.flush() | ||||
assert s == {} | assert s == {} | ||||
Show All 10 Lines | def test_buffering_proxy_storage_content_threshold_bytes_hit(sample_data): | ||||
) | ) | ||||
assert contents[0]['length'] > content_bytes_min_batch_size | assert contents[0]['length'] > content_bytes_min_batch_size | ||||
s = storage.content_add([contents[0]]) | s = storage.content_add([contents[0]]) | ||||
assert s == { | assert s == { | ||||
'content:add': 1, | 'content:add': 1, | ||||
'content:add:bytes': contents[0]['length'], | 'content:add:bytes': contents[0]['length'], | ||||
'skipped_content:add': 0 | |||||
} | } | ||||
missing_contents = storage.content_missing([contents[0]]) | missing_contents = storage.content_missing([contents[0]]) | ||||
assert list(missing_contents) == [] | assert list(missing_contents) == [] | ||||
s = storage.flush() | s = storage.flush() | ||||
assert s == {} | assert s == {} | ||||
def test_buffering_proxy_storage_skipped_content_threshold_not_hit( | |||||
sample_data): | |||||
contents = sample_data['skipped_content'] | |||||
storage = BufferingProxyStorage( | |||||
storage={'cls': 'memory'}, | |||||
min_batch_size={ | |||||
'skipped_content': 10, | |||||
} | |||||
) | |||||
s = storage.skipped_content_add([contents[0], contents[1]]) | |||||
assert s == {} | |||||
# contents have not been written to storage | |||||
missing_contents = storage.skipped_content_missing( | |||||
[contents[0], contents[1]]) | |||||
assert {c['sha1'] for c in missing_contents} \ | |||||
== {c['sha1'] for c in contents} | |||||
s = storage.flush() | |||||
assert s == { | |||||
'skipped_content:add': 1 + 1 | |||||
} | |||||
missing_contents = storage.skipped_content_missing( | |||||
[contents[0], contents[1]]) | |||||
assert list(missing_contents) == [] | |||||
def test_buffering_proxy_storage_skipped_content_threshold_nb_hit(sample_data): | |||||
contents = sample_data['skipped_content'] | |||||
storage = BufferingProxyStorage( | |||||
storage={'cls': 'memory'}, | |||||
min_batch_size={ | |||||
'skipped_content': 1, | |||||
} | |||||
) | |||||
s = storage.skipped_content_add([contents[0]]) | |||||
assert s == { | |||||
'skipped_content:add': 1 | |||||
} | |||||
missing_contents = storage.skipped_content_missing([contents[0]]) | |||||
assert list(missing_contents) == [] | |||||
s = storage.flush() | |||||
assert s == {} | |||||
def test_buffering_proxy_storage_directory_threshold_not_hit(sample_data): | def test_buffering_proxy_storage_directory_threshold_not_hit(sample_data): | ||||
directories = sample_data['directory'] | directories = sample_data['directory'] | ||||
storage = BufferingProxyStorage( | storage = BufferingProxyStorage( | ||||
storage={'cls': 'memory'}, | storage={'cls': 'memory'}, | ||||
min_batch_size={ | min_batch_size={ | ||||
'directory': 10, | 'directory': 10, | ||||
} | } | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 136 Lines • Show Last 20 Lines |