Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_filter.py
| # Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from swh.storage.filter import FilteringProxyStorage | from swh.storage import get_storage | ||||
| storage_config = { | storage_config = { | ||||
| 'cls': 'validate', | 'cls': 'pipeline', | ||||
| 'storage': { | 'steps': [ | ||||
| 'cls': 'memory' | {'cls': 'validate'}, | ||||
| } | {'cls': 'filter'}, | ||||
| {'cls': 'memory'}, | |||||
| ] | |||||
| } | } | ||||
| def test_filtering_proxy_storage_content(sample_data): | def test_filtering_proxy_storage_content(sample_data): | ||||
| sample_content = sample_data['content'][0] | sample_content = sample_data['content'][0] | ||||
| storage = FilteringProxyStorage(storage=storage_config) | storage = get_storage(**storage_config) | ||||
| content = next(storage.content_get([sample_content['sha1']])) | content = next(storage.content_get([sample_content['sha1']])) | ||||
| assert not content | assert not content | ||||
| s = storage.content_add([sample_content]) | s = storage.content_add([sample_content]) | ||||
| assert s == { | assert s == { | ||||
| 'content:add': 1, | 'content:add': 1, | ||||
| 'content:add:bytes': sample_content['length'], | 'content:add:bytes': sample_content['length'], | ||||
| } | } | ||||
| content = next(storage.content_get([sample_content['sha1']])) | content = next(storage.content_get([sample_content['sha1']])) | ||||
| assert content is not None | assert content is not None | ||||
| s = storage.content_add([sample_content]) | s = storage.content_add([sample_content]) | ||||
| assert s == { | assert s == { | ||||
| 'content:add': 0, | 'content:add': 0, | ||||
| 'content:add:bytes': 0, | 'content:add:bytes': 0, | ||||
| } | } | ||||
| def test_filtering_proxy_storage_skipped_content(sample_data): | def test_filtering_proxy_storage_skipped_content(sample_data): | ||||
| sample_content = sample_data['skipped_content'][0] | sample_content = sample_data['skipped_content'][0] | ||||
| storage = FilteringProxyStorage(storage=storage_config) | storage = get_storage(**storage_config) | ||||
| content = next(storage.skipped_content_missing([sample_content])) | content = next(storage.skipped_content_missing([sample_content])) | ||||
| assert content['sha1'] == sample_content['sha1'] | assert content['sha1'] == sample_content['sha1'] | ||||
| s = storage.skipped_content_add([sample_content]) | s = storage.skipped_content_add([sample_content]) | ||||
| assert s == { | assert s == { | ||||
| 'skipped_content:add': 1, | 'skipped_content:add': 1, | ||||
| } | } | ||||
| content = list(storage.skipped_content_missing([sample_content])) | content = list(storage.skipped_content_missing([sample_content])) | ||||
| assert content == [] | assert content == [] | ||||
| s = storage.skipped_content_add([sample_content]) | s = storage.skipped_content_add([sample_content]) | ||||
| assert s == { | assert s == { | ||||
| 'skipped_content:add': 0, | 'skipped_content:add': 0, | ||||
| } | } | ||||
| def test_filtering_proxy_storage_skipped_content_missing_sha1_git(sample_data): | def test_filtering_proxy_storage_skipped_content_missing_sha1_git(sample_data): | ||||
| sample_content = sample_data['skipped_content'][0] | sample_content = sample_data['skipped_content'][0] | ||||
| sample_content2 = sample_data['skipped_content'][1] | sample_content2 = sample_data['skipped_content'][1] | ||||
| storage = FilteringProxyStorage(storage=storage_config) | storage = get_storage(**storage_config) | ||||
| sample_content['sha1_git'] = sample_content2['sha1_git'] = None | sample_content['sha1_git'] = sample_content2['sha1_git'] = None | ||||
| content = next(storage.skipped_content_missing([sample_content])) | content = next(storage.skipped_content_missing([sample_content])) | ||||
| assert content['sha1'] == sample_content['sha1'] | assert content['sha1'] == sample_content['sha1'] | ||||
| s = storage.skipped_content_add([sample_content]) | s = storage.skipped_content_add([sample_content]) | ||||
| assert s == { | assert s == { | ||||
| 'skipped_content:add': 1, | 'skipped_content:add': 1, | ||||
| } | } | ||||
| content = list(storage.skipped_content_missing([sample_content])) | content = list(storage.skipped_content_missing([sample_content])) | ||||
| assert content == [] | assert content == [] | ||||
| s = storage.skipped_content_add([sample_content2]) | s = storage.skipped_content_add([sample_content2]) | ||||
| assert s == { | assert s == { | ||||
| 'skipped_content:add': 1, | 'skipped_content:add': 1, | ||||
| } | } | ||||
| content = list(storage.skipped_content_missing([sample_content2])) | content = list(storage.skipped_content_missing([sample_content2])) | ||||
| assert content == [] | assert content == [] | ||||
| def test_filtering_proxy_storage_revision(sample_data): | def test_filtering_proxy_storage_revision(sample_data): | ||||
| sample_revision = sample_data['revision'][0] | sample_revision = sample_data['revision'][0] | ||||
| storage = FilteringProxyStorage(storage=storage_config) | storage = get_storage(**storage_config) | ||||
| revision = next(storage.revision_get([sample_revision['id']])) | revision = next(storage.revision_get([sample_revision['id']])) | ||||
| assert not revision | assert not revision | ||||
| s = storage.revision_add([sample_revision]) | s = storage.revision_add([sample_revision]) | ||||
| assert s == { | assert s == { | ||||
| 'revision:add': 1, | 'revision:add': 1, | ||||
| } | } | ||||
| revision = next(storage.revision_get([sample_revision['id']])) | revision = next(storage.revision_get([sample_revision['id']])) | ||||
| assert revision is not None | assert revision is not None | ||||
| s = storage.revision_add([sample_revision]) | s = storage.revision_add([sample_revision]) | ||||
| assert s == { | assert s == { | ||||
| 'revision:add': 0, | 'revision:add': 0, | ||||
| } | } | ||||
| def test_filtering_proxy_storage_directory(sample_data): | def test_filtering_proxy_storage_directory(sample_data): | ||||
| sample_directory = sample_data['directory'][0] | sample_directory = sample_data['directory'][0] | ||||
| storage = FilteringProxyStorage(storage=storage_config) | storage = get_storage(**storage_config) | ||||
| directory = next(storage.directory_missing([sample_directory['id']])) | directory = next(storage.directory_missing([sample_directory['id']])) | ||||
| assert directory | assert directory | ||||
| s = storage.directory_add([sample_directory]) | s = storage.directory_add([sample_directory]) | ||||
| assert s == { | assert s == { | ||||
| 'directory:add': 1, | 'directory:add': 1, | ||||
| } | } | ||||
| directory = list(storage.directory_missing([sample_directory['id']])) | directory = list(storage.directory_missing([sample_directory['id']])) | ||||
| assert not directory | assert not directory | ||||
| s = storage.directory_add([sample_directory]) | s = storage.directory_add([sample_directory]) | ||||
| assert s == { | assert s == { | ||||
| 'directory:add': 0, | 'directory:add': 0, | ||||
| } | } | ||||