Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/tests/test_language.py
| # Copyright (C) 2017-2018 The Software Heritage developers | # Copyright (C) 2017-2018 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import unittest | import unittest | ||||
| from swh.indexer import language | from swh.indexer import language | ||||
| from swh.indexer.language import LanguageIndexer | from swh.indexer.language import LanguageIndexer | ||||
| from swh.indexer.tests.test_utils import ( | from swh.indexer.tests.test_utils import ( | ||||
| BasicMockIndexerStorage, MockObjStorage, CommonContentIndexerTest, | CommonContentIndexerTest, CommonIndexerWithErrorsTest, | ||||
| CommonIndexerWithErrorsTest, CommonIndexerNoTool, BASE_TEST_CONFIG | CommonIndexerNoTool, BASE_TEST_CONFIG, fill_storage, fill_obj_storage | ||||
| ) | ) | ||||
| class LanguageTestIndexer(LanguageIndexer): | class LanguageTestIndexer(LanguageIndexer): | ||||
| """Specific language whose configuration is enough to satisfy the | """Specific language whose configuration is enough to satisfy the | ||||
| indexing tests. | indexing tests. | ||||
| """ | """ | ||||
| def parse_config_file(self, *args, **kwargs): | def parse_config_file(self, *args, **kwargs): | ||||
| return { | return { | ||||
| **BASE_TEST_CONFIG, | **BASE_TEST_CONFIG, | ||||
| 'tools': { | 'tools': { | ||||
| 'name': 'pygments', | 'name': 'pygments', | ||||
| 'version': '2.0.1+dfsg-1.1+deb8u1', | 'version': '2.0.1+dfsg-1.1+deb8u1', | ||||
| 'configuration': { | 'configuration': { | ||||
| 'type': 'library', | 'type': 'library', | ||||
| 'debian-package': 'python3-pygments', | 'debian-package': 'python3-pygments', | ||||
| 'max_content_size': 10240, | 'max_content_size': 10240, | ||||
| }, | }, | ||||
| } | } | ||||
| } | } | ||||
| def prepare(self): | |||||
| super().prepare() | |||||
| self.idx_storage = BasicMockIndexerStorage() | |||||
| self.objstorage = MockObjStorage() | |||||
| self.tool_config = self.config['tools']['configuration'] | |||||
| class Language(unittest.TestCase): | class Language(unittest.TestCase): | ||||
| """Tests pygments tool for language detection | """Tests pygments tool for language detection | ||||
| """ | """ | ||||
| def test_compute_language_none(self): | def test_compute_language_none(self): | ||||
| # given | # given | ||||
| self.content = "" | self.content = "" | ||||
| self.declared_language = { | self.declared_language = { | ||||
| 'lang': None | 'lang': None | ||||
| } | } | ||||
| # when | # when | ||||
| result = language.compute_language(self.content) | result = language.compute_language(self.content) | ||||
| # then | # then | ||||
| self.assertEqual(self.declared_language, result) | self.assertEqual(self.declared_language, result) | ||||
| class TestLanguageIndexer(CommonContentIndexerTest, unittest.TestCase): | class TestLanguageIndexer(CommonContentIndexerTest, unittest.TestCase): | ||||
| """Language indexer test scenarios: | """Language indexer test scenarios: | ||||
| - Known sha1s in the input list have their data indexed | - Known sha1s in the input list have their data indexed | ||||
| - Unknown sha1 in the input list are not indexed | - Unknown sha1 in the input list are not indexed | ||||
| """ | """ | ||||
| def get_indexer_results(self, ids): | |||||
| yield from self.indexer.idx_storage.content_language_get(ids) | |||||
| def setUp(self): | def setUp(self): | ||||
| self.indexer = LanguageTestIndexer() | self.indexer = LanguageTestIndexer() | ||||
| fill_storage(self.indexer.storage) | |||||
| fill_obj_storage(self.indexer.objstorage) | |||||
| self.id0 = '02fb2c89e14f7fab46701478c83779c7beb7b069' | self.id0 = '02fb2c89e14f7fab46701478c83779c7beb7b069' | ||||
| self.id1 = '103bc087db1d26afc3a0283f38663d081e9b01e6' | self.id1 = '103bc087db1d26afc3a0283f38663d081e9b01e6' | ||||
| self.id2 = 'd4c647f0fc257591cc9ba1722484229780d1c607' | self.id2 = 'd4c647f0fc257591cc9ba1722484229780d1c607' | ||||
| tool_id = self.indexer.tool['id'] | tool_id = self.indexer.tool['id'] | ||||
| self.expected_results = { | self.expected_results = { | ||||
| self.id0: { | self.id0: { | ||||
| Show All 26 Lines | |||||