diff --git a/swh/indexer/sql/50-swh-data.sql b/swh/indexer/sql/50-swh-data.sql --- a/swh/indexer/sql/50-swh-data.sql +++ b/swh/indexer/sql/50-swh-data.sql @@ -1,26 +0,0 @@ -insert into indexer_configuration(tool_name, tool_version, tool_configuration) -values ('nomos', '3.1.0rc2-31-ga2cbb8c', '{"command_line": "nomossa "}'); - -insert into indexer_configuration(tool_name, tool_version, tool_configuration) -values ('file', '5.22', '{"command_line": "file --mime "}'); - -insert into indexer_configuration(tool_name, tool_version, tool_configuration) -values ('universal-ctags', '~git7859817b', '{"command_line": "ctags --fields=+lnz --sort=no --links=no --output-format=json "}'); - -insert into indexer_configuration(tool_name, tool_version, tool_configuration) -values ('pygments', '2.0.1+dfsg-1.1+deb8u1', '{"type": "library", "debian-package": "python3-pygments"}'); - -insert into indexer_configuration(tool_name, tool_version, tool_configuration) -values ('pygments', '2.0.1+dfsg-1.1+deb8u1', '{"type": "library", "debian-package": "python3-pygments", "max_content_size": 10240}'); - -insert into indexer_configuration(tool_name, tool_version, tool_configuration) -values ('swh-metadata-translator', '0.0.1', '{"type": "local", "context": "NpmMapping"}'); - -insert into indexer_configuration(tool_name, tool_version, tool_configuration) -values ('swh-metadata-detector', '0.0.1', '{"type": "local", "context": ["NpmMapping", "CodemetaMapping"]}'); - -insert into indexer_configuration(tool_name, tool_version, tool_configuration) -values ('swh-deposit', '0.0.1', '{"sword_version": "2"}'); - -insert into indexer_configuration(tool_name, tool_version, tool_configuration) -values ('file', '1:5.30-1+deb9u1', '{"type": "library", "debian-package": "python3-magic"}'); diff --git a/swh/indexer/storage/in_memory.py b/swh/indexer/storage/in_memory.py --- a/swh/indexer/storage/in_memory.py +++ b/swh/indexer/storage/in_memory.py @@ -3,6 +3,8 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import json + class IndexerStorage: """In-memory SWH indexer storage.""" @@ -56,4 +58,4 @@ def _tool_key(self, tool): return (tool['tool_name'], tool['tool_version'], - tuple(sorted(tool['tool_configuration'].items()))) + json.dumps(tool['tool_configuration'], sort_keys=True)) diff --git a/swh/indexer/tests/storage/test_in_memory.py b/swh/indexer/tests/storage/test_in_memory.py --- a/swh/indexer/tests/storage/test_in_memory.py +++ b/swh/indexer/tests/storage/test_in_memory.py @@ -1,15 +1,17 @@ from unittest import TestCase import pytest -from swh.indexer.storage.in_memory import IndexerStorage - -from .test_storage import CommonTestStorage, CommonPropBasedTestStorage +from .test_storage import CommonTestStorage class IndexerTestInMemoryStorage(CommonTestStorage, TestCase): def setUp(self): + self.storage_config = { + 'cls': 'memory', + 'args': { + }, + } super().setUp() - self.storage = IndexerStorage() @pytest.mark.xfail def test_check_config(self): @@ -143,12 +145,6 @@ def test_indexer_configuration_metadata_get(self): pass - -class PropIndexerTestInMemoryStorage(CommonPropBasedTestStorage, TestCase): - def setUp(self): - super().setUp() - self.storage = IndexerStorage() - @pytest.mark.xfail def test_generate_content_mimetype_get_range_limit_none(self): pass diff --git a/swh/indexer/tests/storage/test_storage.py b/swh/indexer/tests/storage/test_storage.py --- a/swh/indexer/tests/storage/test_storage.py +++ b/swh/indexer/tests/storage/test_storage.py @@ -18,6 +18,52 @@ ) from swh.indexer.tests.storage import SQL_DIR +TOOLS = [ + { + 'tool_name': 'universal-ctags', + 'tool_version': '~git7859817b', + 'tool_configuration': { + "command_line": "ctags --fields=+lnz --sort=no --links=no " + "--output-format=json "} + }, + { + 'tool_name': 'swh-metadata-translator', + 'tool_version': '0.0.1', + 'tool_configuration': {"type": "local", "context": "NpmMapping"}, + }, + { + 'tool_name': 'swh-metadata-detector', + 'tool_version': '0.0.1', + 'tool_configuration': { + "type": "local", "context": ["NpmMapping", "CodemetaMapping"]}, + }, + { + 'tool_name': 'file', + 'tool_version': '5.22', + 'tool_configuration': {"command_line": "file --mime "}, + }, + { + 'tool_name': 'pygments', + 'tool_version': '2.0.1+dfsg-1.1+deb8u1', + 'tool_configuration': { + "type": "library", "debian-package": "python3-pygments"}, + }, + { + 'tool_name': 'pygments', + 'tool_version': '2.0.1+dfsg-1.1+deb8u1', + 'tool_configuration': { + "type": "library", + "debian-package": "python3-pygments", + "max_content_size": 10240 + }, + }, + { + 'tool_name': 'nomos', + 'tool_version': '3.1.0rc2-31-ga2cbb8c', + 'tool_configuration': {"command_line": "nomossa "}, + } +] + @pytest.mark.db class BasePgTestStorage(SingleDbTestFixture): @@ -38,7 +84,6 @@ 'db': 'dbname=%s' % self.TEST_DB_NAME, }, } - self.storage = get_indexer_storage(**self.storage_config) self.sha1_1 = hash_to_bytes('34973274ccef6ab4dfaaf86599792fa9c3fe4689') self.sha1_2 = hash_to_bytes('61c2b3a30496d329e21af70dd2d7e097046d07b7') @@ -49,24 +94,6 @@ self.origin_id_1 = 54974445 self.origin_id_2 = 44434342 - cur = self.test_db[self.TEST_DB_NAME].cursor - tools = {} - cur.execute(''' - select tool_name, id, tool_version, tool_configuration - from indexer_configuration - order by id''') - for row in cur.fetchall(): - key = row[0] - while key in tools: - key = '_' + key - tools[key] = { - 'id': row[1], - 'name': row[0], - 'version': row[2], - 'configuration': row[3] - } - self.tools = tools - def tearDown(self): self.reset_storage_tables() self.storage = None @@ -84,6 +111,21 @@ """Base class for Indexer Storage testing. """ + def setUp(self): + super().setUp() + self.storage = get_indexer_storage(**self.storage_config) + tools = self.storage.indexer_configuration_add(TOOLS) + self.tools = {} + for tool in tools: + tool_name = tool['tool_name'] + while tool_name in self.tools: + tool_name += '_' + self.tools[tool_name] = { + 'id': tool['id'], + 'name': tool['tool_name'], + 'version': tool['tool_version'], + 'configuration': tool['tool_configuration'], + } def test_check_config(self): self.assertTrue(self.storage.check_config(check_write=True)) @@ -213,7 +255,7 @@ 'mimetype': 'text/html', 'encoding': 'us-ascii', 'tool': { - 'id': 2, + 'id': self.tools['file']['id'], 'name': 'file', 'version': '5.22', 'configuration': {'command_line': 'file --mime '} @@ -1740,12 +1782,7 @@ self.assertEqual(expected_tool, actual_tool) - -@pytest.mark.property_based -class CommonPropBasedTestStorage(BasePgTestStorage): - """Properties-based tests - - """ + @pytest.mark.property_based def test_generate_content_mimetype_get_range_limit_none(self): """mimetype_get_range call with wrong limit input should fail""" with self.assertRaises(ValueError) as e: @@ -1756,6 +1793,7 @@ self.assertEqual(e.exception.args, ( 'Development error: limit should not be None',)) + @pytest.mark.property_based @given(gen_content_mimetypes(min_size=1, max_size=4)) def test_generate_content_mimetype_get_range_no_limit(self, mimetypes): """mimetype_get_range returns mimetypes within range provided""" @@ -1781,6 +1819,7 @@ self.assertIsNone(actual_next) self.assertEqual(content_ids, actual_ids) + @pytest.mark.property_based @given(gen_content_mimetypes(min_size=4, max_size=4)) def test_generate_content_mimetype_get_range_limit(self, mimetypes): """mimetype_get_range paginates results if limit exceeded""" @@ -1821,6 +1860,7 @@ expected_mimetypes2 = [content_ids[-1]] self.assertEqual(expected_mimetypes2, actual_ids2) + @pytest.mark.property_based def test_generate_content_fossology_license_get_range_limit_none(self): """license_get_range call with wrong limit input should fail""" with self.assertRaises(ValueError) as e: @@ -1831,6 +1871,7 @@ self.assertEqual(e.exception.args, ( 'Development error: limit should not be None',)) + @pytest.mark.property_based def prepare_mimetypes_from(self, fossology_licenses): """Fossology license needs some consistent data in db to run. @@ -1845,6 +1886,7 @@ }) return mimetypes + @pytest.mark.property_based @given(gen_content_fossology_licenses(min_size=1, max_size=4)) def test_generate_content_fossology_license_get_range_no_limit( self, fossology_licenses): @@ -1875,6 +1917,7 @@ self.assertIsNone(actual_next) self.assertEqual(content_ids, actual_ids) + @pytest.mark.property_based @given(gen_content_fossology_licenses(min_size=1, max_size=4), gen_content_mimetypes(min_size=1, max_size=1)) def test_generate_content_fossology_license_get_range_no_limit_with_filter( @@ -1913,6 +1956,7 @@ self.assertIsNone(actual_next) self.assertEqual(content_ids, actual_ids) + @pytest.mark.property_based @given(gen_content_fossology_licenses(min_size=4, max_size=4)) def test_generate_fossology_license_get_range_limit( self, fossology_licenses): @@ -1969,9 +2013,3 @@ """ pass - - -@pytest.mark.property_based -class PropIndexerTestStorage(CommonPropBasedTestStorage, BasePgTestStorage, - unittest.TestCase): - pass