Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/tests/storage/test_storage.py
| Show All 33 Lines | TOOLS = [ | ||||
| }, | }, | ||||
| { | { | ||||
| 'tool_name': 'swh-metadata-detector', | 'tool_name': 'swh-metadata-detector', | ||||
| 'tool_version': '0.0.1', | 'tool_version': '0.0.1', | ||||
| 'tool_configuration': { | 'tool_configuration': { | ||||
| "type": "local", "context": ["NpmMapping", "CodemetaMapping"]}, | "type": "local", "context": ["NpmMapping", "CodemetaMapping"]}, | ||||
| }, | }, | ||||
| { | { | ||||
| 'tool_name': 'swh-metadata-detector2', | |||||
| 'tool_version': '0.0.1', | |||||
| 'tool_configuration': { | |||||
| "type": "local", "context": ["NpmMapping", "CodemetaMapping"]}, | |||||
| }, | |||||
| { | |||||
| 'tool_name': 'file', | 'tool_name': 'file', | ||||
| 'tool_version': '5.22', | 'tool_version': '5.22', | ||||
| 'tool_configuration': {"command_line": "file --mime <filepath>"}, | 'tool_configuration': {"command_line": "file --mime <filepath>"}, | ||||
| }, | }, | ||||
| { | { | ||||
| 'tool_name': 'pygments', | 'tool_name': 'pygments', | ||||
| 'tool_version': '2.0.1+dfsg-1.1+deb8u1', | 'tool_version': '2.0.1+dfsg-1.1+deb8u1', | ||||
| 'tool_configuration': { | 'tool_configuration': { | ||||
| ▲ Show 20 Lines • Show All 1,119 Lines • ▼ Show 20 Lines | def test_origin_intrinsic_metadata_search_fulltext_rank(self): | ||||
| self.assertEqual( | self.assertEqual( | ||||
| [res['origin_id'] for res in search(['Jane'])], | [res['origin_id'] for res in search(['Jane'])], | ||||
| [self.origin_id_2, self.origin_id_1]) | [self.origin_id_2, self.origin_id_1]) | ||||
| self.assertEqual( | self.assertEqual( | ||||
| [res['origin_id'] for res in search(['John', 'Jane'])], | [res['origin_id'] for res in search(['John', 'Jane'])], | ||||
| [self.origin_id_1]) | [self.origin_id_1]) | ||||
| def _fill_origin_intrinsic_metadata(self): | def _fill_origin_intrinsic_metadata(self): | ||||
| tool_id = self.tools['swh-metadata-detector']['id'] | tool1_id = self.tools['swh-metadata-detector']['id'] | ||||
| tool2_id = self.tools['swh-metadata-detector2']['id'] | |||||
| metadata1 = { | metadata1 = { | ||||
| '@context': 'foo', | '@context': 'foo', | ||||
| 'author': 'John Doe', | 'author': 'John Doe', | ||||
| } | } | ||||
| metadata1_rev = { | metadata1_rev = { | ||||
| 'id': self.revision_id_1, | 'id': self.revision_id_1, | ||||
| 'translated_metadata': metadata1, | 'translated_metadata': metadata1, | ||||
| 'mappings': ['npm'], | 'mappings': ['npm'], | ||||
| 'indexer_configuration_id': tool_id, | 'indexer_configuration_id': tool1_id, | ||||
| } | } | ||||
| metadata1_origin = { | metadata1_origin = { | ||||
| 'origin_id': self.origin_id_1, | 'origin_id': self.origin_id_1, | ||||
| 'metadata': metadata1, | 'metadata': metadata1, | ||||
| 'mappings': ['npm'], | 'mappings': ['npm'], | ||||
| 'indexer_configuration_id': tool_id, | 'indexer_configuration_id': tool1_id, | ||||
| 'from_revision': self.revision_id_1, | 'from_revision': self.revision_id_1, | ||||
| } | } | ||||
| metadata2 = { | metadata2 = { | ||||
| '@context': 'foo', | '@context': 'foo', | ||||
| 'author': 'Jane Doe', | 'author': 'Jane Doe', | ||||
| } | } | ||||
| metadata2_rev = { | metadata2_rev = { | ||||
| 'id': self.revision_id_2, | 'id': self.revision_id_2, | ||||
| 'translated_metadata': metadata2, | 'translated_metadata': metadata2, | ||||
| 'mappings': ['npm', 'gemspec'], | 'mappings': ['npm', 'gemspec'], | ||||
| 'indexer_configuration_id': tool_id, | 'indexer_configuration_id': tool2_id, | ||||
| } | } | ||||
| metadata2_origin = { | metadata2_origin = { | ||||
| 'origin_id': self.origin_id_2, | 'origin_id': self.origin_id_2, | ||||
| 'metadata': metadata2, | 'metadata': metadata2, | ||||
| 'mappings': ['npm', 'gemspec'], | 'mappings': ['npm', 'gemspec'], | ||||
| 'indexer_configuration_id': tool_id, | 'indexer_configuration_id': tool2_id, | ||||
| 'from_revision': self.revision_id_2, | 'from_revision': self.revision_id_2, | ||||
| } | } | ||||
| metadata3 = { | metadata3 = { | ||||
| '@context': 'foo', | '@context': 'foo', | ||||
| } | } | ||||
| metadata3_rev = { | metadata3_rev = { | ||||
| 'id': self.revision_id_3, | 'id': self.revision_id_3, | ||||
| 'translated_metadata': metadata3, | 'translated_metadata': metadata3, | ||||
| 'mappings': ['npm', 'gemspec'], | 'mappings': ['npm', 'gemspec'], | ||||
| 'indexer_configuration_id': tool_id, | 'indexer_configuration_id': tool2_id, | ||||
| } | } | ||||
| metadata3_origin = { | metadata3_origin = { | ||||
| 'origin_id': self.origin_id_3, | 'origin_id': self.origin_id_3, | ||||
| 'metadata': metadata3, | 'metadata': metadata3, | ||||
| 'mappings': ['pkg-info'], | 'mappings': ['pkg-info'], | ||||
| 'indexer_configuration_id': tool_id, | 'indexer_configuration_id': tool2_id, | ||||
| 'from_revision': self.revision_id_3, | 'from_revision': self.revision_id_3, | ||||
| } | } | ||||
| self.storage.revision_metadata_add([metadata1_rev]) | self.storage.revision_metadata_add([metadata1_rev]) | ||||
| self.storage.origin_intrinsic_metadata_add([metadata1_origin]) | self.storage.origin_intrinsic_metadata_add([metadata1_origin]) | ||||
| self.storage.revision_metadata_add([metadata2_rev]) | self.storage.revision_metadata_add([metadata2_rev]) | ||||
| self.storage.origin_intrinsic_metadata_add([metadata2_origin]) | self.storage.origin_intrinsic_metadata_add([metadata2_origin]) | ||||
| self.storage.revision_metadata_add([metadata3_rev]) | self.storage.revision_metadata_add([metadata3_rev]) | ||||
| self.storage.origin_intrinsic_metadata_add([metadata3_origin]) | self.storage.origin_intrinsic_metadata_add([metadata3_origin]) | ||||
| def test_origin_intrinsic_metadata_search_by_producer(self): | def test_origin_intrinsic_metadata_search_by_producer(self): | ||||
| self._fill_origin_intrinsic_metadata() | self._fill_origin_intrinsic_metadata() | ||||
| tool = self.tools['swh-metadata-detector'] | tool1 = self.tools['swh-metadata-detector'] | ||||
| tool2 = self.tools['swh-metadata-detector2'] | |||||
| endpoint = self.storage.origin_intrinsic_metadata_search_by_producer | endpoint = self.storage.origin_intrinsic_metadata_search_by_producer | ||||
| # test pagination | # test pagination | ||||
| self.assertCountEqual( | self.assertCountEqual( | ||||
| endpoint(ids_only=True), | endpoint(ids_only=True), | ||||
| [self.origin_id_1, self.origin_id_2, self.origin_id_3]) | [self.origin_id_1, self.origin_id_2, self.origin_id_3]) | ||||
| self.assertCountEqual( | self.assertCountEqual( | ||||
| endpoint(start=0, ids_only=True), | endpoint(start=0, ids_only=True), | ||||
| Show All 26 Lines | def test_origin_intrinsic_metadata_search_by_producer(self): | ||||
| endpoint(mappings=['foobar'], ids_only=True), | endpoint(mappings=['foobar'], ids_only=True), | ||||
| []) | []) | ||||
| # test pagination + mappings | # test pagination + mappings | ||||
| self.assertCountEqual( | self.assertCountEqual( | ||||
| endpoint(mappings=['npm'], limit=1, ids_only=True), | endpoint(mappings=['npm'], limit=1, ids_only=True), | ||||
| [self.origin_id_1]) | [self.origin_id_1]) | ||||
| # test tool filtering | |||||
| self.assertCountEqual( | |||||
| endpoint(tool_ids=[tool1['id']], ids_only=True), | |||||
| [self.origin_id_1]) | |||||
| self.assertCountEqual( | |||||
| endpoint(tool_ids=[tool2['id']], ids_only=True), | |||||
| [self.origin_id_2, self.origin_id_3]) | |||||
| self.assertCountEqual( | |||||
| endpoint(tool_ids=[tool1['id'], tool2['id']], ids_only=True), | |||||
| [self.origin_id_1, self.origin_id_2, self.origin_id_3]) | |||||
| # test ids_only=False | # test ids_only=False | ||||
| self.assertEqual(list(endpoint(mappings=['gemspec'])), [{ | self.assertEqual(list(endpoint(mappings=['gemspec'])), [{ | ||||
| 'origin_id': self.origin_id_2, | 'origin_id': self.origin_id_2, | ||||
| 'metadata': { | 'metadata': { | ||||
| '@context': 'foo', | '@context': 'foo', | ||||
| 'author': 'Jane Doe', | 'author': 'Jane Doe', | ||||
| }, | }, | ||||
| 'mappings': ['npm', 'gemspec'], | 'mappings': ['npm', 'gemspec'], | ||||
| 'tool': tool, | 'tool': tool2, | ||||
| 'from_revision': self.revision_id_2, | 'from_revision': self.revision_id_2, | ||||
| }]) | }]) | ||||
| def test_origin_intrinsic_metadata_stats(self): | def test_origin_intrinsic_metadata_stats(self): | ||||
| self._fill_origin_intrinsic_metadata() | self._fill_origin_intrinsic_metadata() | ||||
| result = self.storage.origin_intrinsic_metadata_stats() | result = self.storage.origin_intrinsic_metadata_stats() | ||||
| self.assertEqual(result, { | self.assertEqual(result, { | ||||
| ▲ Show 20 Lines • Show All 350 Lines • Show Last 20 Lines | |||||