diff --git a/swh/indexer/storage/in_memory.py b/swh/indexer/storage/in_memory.py --- a/swh/indexer/storage/in_memory.py +++ b/swh/indexer/storage/in_memory.py @@ -199,6 +199,9 @@ self._revision_intrinsic_metadata = SubStorage(self._tools) self._origin_intrinsic_metadata = SubStorage(self._tools) + def check_config(self, *, check_write): + return True + def content_mimetype_missing(self, mimetypes): """Generate mimetypes missing from storage. diff --git a/swh/indexer/tests/storage/conftest.py b/swh/indexer/tests/storage/conftest.py --- a/swh/indexer/tests/storage/conftest.py +++ b/swh/indexer/tests/storage/conftest.py @@ -10,63 +10,13 @@ from swh.storage.tests.conftest import postgresql_fact from swh.indexer.storage import get_indexer_storage from swh.model.hashutil import hash_to_bytes -from .generate_data_test import MIMETYPE_OBJECTS, FOSSOLOGY_LICENSES +from .generate_data_test import ( + MIMETYPE_OBJECTS, FOSSOLOGY_LICENSES, TOOLS +) DUMP_FILES = join(SQL_DIR, '*.sql') -TOOLS = [ - { - 'tool_name': 'universal-ctags', - 'tool_version': '~git7859817b', - 'tool_configuration': { - "command_line": "ctags --fields=+lnz --sort=no --links=no " - "--output-format=json "} - }, - { - 'tool_name': 'swh-metadata-translator', - 'tool_version': '0.0.1', - 'tool_configuration': {"type": "local", "context": "NpmMapping"}, - }, - { - 'tool_name': 'swh-metadata-detector', - 'tool_version': '0.0.1', - 'tool_configuration': { - "type": "local", "context": ["NpmMapping", "CodemetaMapping"]}, - }, - { - 'tool_name': 'swh-metadata-detector2', - 'tool_version': '0.0.1', - 'tool_configuration': { - "type": "local", "context": ["NpmMapping", "CodemetaMapping"]}, - }, - { - 'tool_name': 'file', - 'tool_version': '5.22', - 'tool_configuration': {"command_line": "file --mime "}, - }, - { - 'tool_name': 'pygments', - 'tool_version': '2.0.1+dfsg-1.1+deb8u1', - 'tool_configuration': { - "type": "library", "debian-package": "python3-pygments"}, - }, - { - 'tool_name': 'pygments2', - 'tool_version': '2.0.1+dfsg-1.1+deb8u1', - 'tool_configuration': { - "type": "library", - "debian-package": "python3-pygments", - "max_content_size": 10240 - }, - }, - { - 'tool_name': 'nomos', - 'tool_version': '3.1.0rc2-31-ga2cbb8c', - 'tool_configuration': {"command_line": "nomossa "}, - } -] - class DataObj(dict): def __getattr__(self, key): @@ -101,10 +51,15 @@ data.origin_url_1 = 'file:///dev/0/zero' # 44434341 data.origin_url_2 = 'file:///dev/1/one' # 44434342 data.origin_url_3 = 'file:///dev/2/two' # 54974445 - data.mimetypes = MIMETYPE_OBJECTS[:] - swh_indexer_storage.content_mimetype_add( - MIMETYPE_OBJECTS) - data.fossology_licenses = FOSSOLOGY_LICENSES[:] + data.mimetypes = [ + {**mimetype_obj, 'indexer_configuration_id': tools['file']['id']} + for mimetype_obj in MIMETYPE_OBJECTS + ] + swh_indexer_storage.content_mimetype_add(data.mimetypes) + data.fossology_licenses = [ + {**fossology_obj, 'indexer_configuration_id': tools['nomos']['id']} + for fossology_obj in FOSSOLOGY_LICENSES + ] swh_indexer_storage._test_data = data return (swh_indexer_storage, data) diff --git a/swh/indexer/tests/storage/generate_data_test.py b/swh/indexer/tests/storage/generate_data_test.py --- a/swh/indexer/tests/storage/generate_data_test.py +++ b/swh/indexer/tests/storage/generate_data_test.py @@ -86,11 +86,64 @@ return content_mimetypes +TOOLS = [ + { + 'tool_name': 'universal-ctags', + 'tool_version': '~git7859817b', + 'tool_configuration': { + "command_line": "ctags --fields=+lnz --sort=no --links=no " + "--output-format=json "} + }, + { + 'tool_name': 'swh-metadata-translator', + 'tool_version': '0.0.1', + 'tool_configuration': {"type": "local", "context": "NpmMapping"}, + }, + { + 'tool_name': 'swh-metadata-detector', + 'tool_version': '0.0.1', + 'tool_configuration': { + "type": "local", "context": ["NpmMapping", "CodemetaMapping"]}, + }, + { + 'tool_name': 'swh-metadata-detector2', + 'tool_version': '0.0.1', + 'tool_configuration': { + "type": "local", "context": ["NpmMapping", "CodemetaMapping"]}, + }, + { + 'tool_name': 'file', + 'tool_version': '5.22', + 'tool_configuration': {"command_line": "file --mime "}, + }, + { + 'tool_name': 'pygments', + 'tool_version': '2.0.1+dfsg-1.1+deb8u1', + 'tool_configuration': { + "type": "library", "debian-package": "python3-pygments"}, + }, + { + 'tool_name': 'pygments2', + 'tool_version': '2.0.1+dfsg-1.1+deb8u1', + 'tool_configuration': { + "type": "library", + "debian-package": "python3-pygments", + "max_content_size": 10240 + }, + }, + { + 'tool_name': 'nomos', + 'tool_version': '3.1.0rc2-31-ga2cbb8c', + 'tool_configuration': {"command_line": "nomossa "}, + }, +] + + MIMETYPE_OBJECTS = [ {'id': MultiHash.from_data(uuid1().bytes, {'sha1'}).digest()['sha1'], - 'indexer_configuration_id': 1, 'mimetype': mt, 'encoding': enc, + # 'indexer_configuration_id' will be added after TOOLS get registered } for mt in MIMETYPES for enc in ENCODINGS] @@ -105,8 +158,8 @@ FOSSOLOGY_LICENSES = [ {'id': MultiHash.from_data(uuid1().bytes, {'sha1'}).digest()['sha1'], - 'indexer_configuration_id': 1, 'licenses': [LICENSES[i % len(LICENSES)], ], + # 'indexer_configuration_id' will be added after TOOLS get registered } for i in range(10) ] diff --git a/swh/indexer/tests/storage/test_in_memory.py b/swh/indexer/tests/storage/test_in_memory.py --- a/swh/indexer/tests/storage/test_in_memory.py +++ b/swh/indexer/tests/storage/test_in_memory.py @@ -11,11 +11,10 @@ @pytest.fixture -def swh_indexer_storage(swh_indexer_storage_postgresql): +def swh_indexer_storage(): storage_config = { - 'cls': 'local', + 'cls': 'memory', 'args': { - 'db': swh_indexer_storage_postgresql.dsn, }, } return get_indexer_storage(**storage_config) diff --git a/swh/indexer/tests/storage/test_storage.py b/swh/indexer/tests/storage/test_storage.py --- a/swh/indexer/tests/storage/test_storage.py +++ b/swh/indexer/tests/storage/test_storage.py @@ -349,6 +349,8 @@ """mimetype_get_range paginates results if limit exceeded""" storage, data = swh_indexer_storage_with_data + indexer_configuration_id = data.tools['file']['id'] + # input the list of sha1s we want from storage content_ids = sorted( [c['id'] for c in data.mimetypes]) @@ -360,7 +362,7 @@ # retrieve mimetypes limited to 10 results actual_result = storage.content_mimetype_get_range( start, end, - indexer_configuration_id=1, + indexer_configuration_id=indexer_configuration_id, limit=10) assert actual_result @@ -377,7 +379,8 @@ # retrieve next part actual_result = storage.content_mimetype_get_range( - start=end, end=end, indexer_configuration_id=1) + start=end, end=end, + indexer_configuration_id=indexer_configuration_id) assert set(actual_result.keys()) == {'ids', 'next'} actual_ids = actual_result['ids'] actual_next = actual_result['next']