Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/tests/storage/test_storage.py
Show All 35 Lines | def setUp(self): | ||||
self.storage = get_indexer_storage(**self.storage_config) | self.storage = get_indexer_storage(**self.storage_config) | ||||
self.sha1_1 = hash_to_bytes('34973274ccef6ab4dfaaf86599792fa9c3fe4689') | self.sha1_1 = hash_to_bytes('34973274ccef6ab4dfaaf86599792fa9c3fe4689') | ||||
self.sha1_2 = hash_to_bytes('61c2b3a30496d329e21af70dd2d7e097046d07b7') | self.sha1_2 = hash_to_bytes('61c2b3a30496d329e21af70dd2d7e097046d07b7') | ||||
self.revision_id_1 = hash_to_bytes( | self.revision_id_1 = hash_to_bytes( | ||||
'7026b7c1a2af56521e951c01ed20f255fa054238') | '7026b7c1a2af56521e951c01ed20f255fa054238') | ||||
self.revision_id_2 = hash_to_bytes( | self.revision_id_2 = hash_to_bytes( | ||||
'7026b7c1a2af56521e9587659012345678904321') | '7026b7c1a2af56521e9587659012345678904321') | ||||
self.origin_id_1 = 54974445 | |||||
cur = self.test_db[self.TEST_DB_NAME].cursor | cur = self.test_db[self.TEST_DB_NAME].cursor | ||||
tools = {} | tools = {} | ||||
cur.execute(''' | cur.execute(''' | ||||
select tool_name, id, tool_version, tool_configuration | select tool_name, id, tool_version, tool_configuration | ||||
from indexer_configuration | from indexer_configuration | ||||
order by id''') | order by id''') | ||||
for row in cur.fetchall(): | for row in cur.fetchall(): | ||||
▲ Show 20 Lines • Show All 1,072 Lines • ▼ Show 20 Lines | def test_revision_metadata_missing(self): | ||||
'operatingSystem': None, | 'operatingSystem': None, | ||||
'description': None, | 'description': None, | ||||
'keywords': None, | 'keywords': None, | ||||
'issueTracker': None, | 'issueTracker': None, | ||||
'name': None, | 'name': None, | ||||
'author': None, | 'author': None, | ||||
'relatedLink': None, | 'relatedLink': None, | ||||
'url': None, | 'url': None, | ||||
'type': None, | |||||
'license': None, | 'license': None, | ||||
'maintainer': None, | 'maintainer': None, | ||||
'email': None, | 'email': None, | ||||
'softwareRequirements': None, | 'softwareRequirements': None, | ||||
'identifier': None | 'identifier': None | ||||
}, | }, | ||||
'indexer_configuration_id': tool_id | 'indexer_configuration_id': tool_id | ||||
}]) | }]) | ||||
Show All 17 Lines | def test_revision_metadata_get(self): | ||||
'operatingSystem': None, | 'operatingSystem': None, | ||||
'description': None, | 'description': None, | ||||
'keywords': None, | 'keywords': None, | ||||
'issueTracker': None, | 'issueTracker': None, | ||||
'name': None, | 'name': None, | ||||
'author': None, | 'author': None, | ||||
'relatedLink': None, | 'relatedLink': None, | ||||
'url': None, | 'url': None, | ||||
'type': None, | |||||
'license': None, | 'license': None, | ||||
'maintainer': None, | 'maintainer': None, | ||||
'email': None, | 'email': None, | ||||
'softwareRequirements': None, | 'softwareRequirements': None, | ||||
'identifier': None | 'identifier': None | ||||
}, | }, | ||||
'indexer_configuration_id': tool_id | 'indexer_configuration_id': tool_id | ||||
} | } | ||||
Show All 25 Lines | def test_revision_metadata_add_drop_duplicate(self): | ||||
'operatingSystem': None, | 'operatingSystem': None, | ||||
'description': None, | 'description': None, | ||||
'keywords': None, | 'keywords': None, | ||||
'issueTracker': None, | 'issueTracker': None, | ||||
'name': None, | 'name': None, | ||||
'author': None, | 'author': None, | ||||
'relatedLink': None, | 'relatedLink': None, | ||||
'url': None, | 'url': None, | ||||
'type': None, | |||||
'license': None, | 'license': None, | ||||
'maintainer': None, | 'maintainer': None, | ||||
'email': None, | 'email': None, | ||||
'softwareRequirements': None, | 'softwareRequirements': None, | ||||
'identifier': None | 'identifier': None | ||||
}, | }, | ||||
'indexer_configuration_id': tool_id, | 'indexer_configuration_id': tool_id, | ||||
} | } | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | def test_revision_metadata_add_update_in_place_duplicate(self): | ||||
'operatingSystem': None, | 'operatingSystem': None, | ||||
'description': None, | 'description': None, | ||||
'keywords': None, | 'keywords': None, | ||||
'issueTracker': None, | 'issueTracker': None, | ||||
'name': None, | 'name': None, | ||||
'author': None, | 'author': None, | ||||
'relatedLink': None, | 'relatedLink': None, | ||||
'url': None, | 'url': None, | ||||
'type': None, | |||||
'license': None, | 'license': None, | ||||
'maintainer': None, | 'maintainer': None, | ||||
'email': None, | 'email': None, | ||||
'softwareRequirements': None, | 'softwareRequirements': None, | ||||
'identifier': None | 'identifier': None | ||||
}, | }, | ||||
'indexer_configuration_id': tool_id, | 'indexer_configuration_id': tool_id, | ||||
} | } | ||||
Show All 21 Lines | def test_revision_metadata_add_update_in_place_duplicate(self): | ||||
'author': 'MG' | 'author': 'MG' | ||||
}, | }, | ||||
}) | }) | ||||
self.storage.revision_metadata_add([metadata_v2], conflict_update=True) | self.storage.revision_metadata_add([metadata_v2], conflict_update=True) | ||||
actual_metadata = list(self.storage.revision_metadata_get( | actual_metadata = list(self.storage.revision_metadata_get( | ||||
[self.revision_id_2])) | [self.revision_id_2])) | ||||
# language did not change as the v2 was dropped. | |||||
expected_metadata_v2 = [{ | expected_metadata_v2 = [{ | ||||
'id': self.revision_id_2, | 'id': self.revision_id_2, | ||||
'translated_metadata': metadata_v2['translated_metadata'], | 'translated_metadata': metadata_v2['translated_metadata'], | ||||
'tool': self.tools['swh-metadata-detector'] | 'tool': self.tools['swh-metadata-detector'] | ||||
}] | }] | ||||
# metadata did change as the v2 was used to overwrite v1 | # metadata did change as the v2 was used to overwrite v1 | ||||
self.assertEqual(actual_metadata, expected_metadata_v2) | self.assertEqual(actual_metadata, expected_metadata_v2) | ||||
def test_origin_intrinsic_metadata_get(self): | |||||
# given | |||||
tool_id = self.tools['swh-metadata-detector']['id'] | |||||
metadata = { | |||||
'developmentStatus': None, | |||||
'version': None, | |||||
'operatingSystem': None, | |||||
'description': None, | |||||
'keywords': None, | |||||
'issueTracker': None, | |||||
'name': None, | |||||
'author': None, | |||||
'relatedLink': None, | |||||
'url': None, | |||||
moranegg: type is no longer in the minimal set.
check all `type` properties in tests | |||||
'license': None, | |||||
'maintainer': None, | |||||
'email': None, | |||||
'softwareRequirements': None, | |||||
'identifier': None, | |||||
} | |||||
metadata_rev = { | |||||
'id': self.revision_id_2, | |||||
'translated_metadata': metadata, | |||||
'indexer_configuration_id': tool_id, | |||||
} | |||||
metadata_origin = { | |||||
'origin_id': self.origin_id_1, | |||||
'metadata': metadata, | |||||
'indexer_configuration_id': tool_id, | |||||
'from_revision': self.revision_id_2, | |||||
} | |||||
# when | |||||
self.storage.revision_metadata_add([metadata_rev]) | |||||
self.storage.origin_intrinsic_metadata_add([metadata_origin]) | |||||
# then | |||||
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( | |||||
[self.origin_id_1, 42])) | |||||
expected_metadata = [{ | |||||
'origin_id': self.origin_id_1, | |||||
'metadata': metadata, | |||||
'tool': self.tools['swh-metadata-detector'], | |||||
'from_revision': self.revision_id_2, | |||||
}] | |||||
self.assertEqual(actual_metadata, expected_metadata) | |||||
def test_origin_intrinsic_metadata_add_drop_duplicate(self): | |||||
# given | |||||
tool_id = self.tools['swh-metadata-detector']['id'] | |||||
metadata_v1 = { | |||||
'developmentStatus': None, | |||||
'version': None, | |||||
'operatingSystem': None, | |||||
'description': None, | |||||
'keywords': None, | |||||
'issueTracker': None, | |||||
'name': None, | |||||
'author': None, | |||||
'relatedLink': None, | |||||
'url': None, | |||||
'license': None, | |||||
'maintainer': None, | |||||
'email': None, | |||||
'softwareRequirements': None, | |||||
'identifier': None | |||||
} | |||||
metadata_rev_v1 = { | |||||
'id': self.revision_id_1, | |||||
'translated_metadata': metadata_v1.copy(), | |||||
'indexer_configuration_id': tool_id, | |||||
} | |||||
metadata_origin_v1 = { | |||||
'origin_id': self.origin_id_1, | |||||
'metadata': metadata_v1.copy(), | |||||
'indexer_configuration_id': tool_id, | |||||
'from_revision': self.revision_id_1, | |||||
} | |||||
# given | |||||
self.storage.revision_metadata_add([metadata_rev_v1]) | |||||
self.storage.origin_intrinsic_metadata_add([metadata_origin_v1]) | |||||
# when | |||||
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( | |||||
[self.origin_id_1, 42])) | |||||
expected_metadata_v1 = [{ | |||||
'origin_id': self.origin_id_1, | |||||
'metadata': metadata_v1, | |||||
'tool': self.tools['swh-metadata-detector'], | |||||
'from_revision': self.revision_id_1, | |||||
}] | |||||
self.assertEqual(actual_metadata, expected_metadata_v1) | |||||
# given | |||||
metadata_v2 = metadata_v1.copy() | |||||
metadata_v2.update({ | |||||
'name': 'test_metadata', | |||||
'author': 'MG', | |||||
}) | |||||
metadata_rev_v2 = metadata_rev_v1.copy() | |||||
metadata_origin_v2 = metadata_origin_v1.copy() | |||||
metadata_rev_v2['translated_metadata'] = metadata_v2 | |||||
metadata_origin_v2['translated_metadata'] = metadata_v2 | |||||
self.storage.revision_metadata_add([metadata_rev_v2]) | |||||
self.storage.origin_intrinsic_metadata_add([metadata_origin_v2]) | |||||
# then | |||||
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( | |||||
[self.origin_id_1])) | |||||
# metadata did not change as the v2 was dropped. | |||||
self.assertEqual(actual_metadata, expected_metadata_v1) | |||||
def test_origin_intrinsic_metadata_add_update_in_place_duplicate(self): | |||||
# given | |||||
tool_id = self.tools['swh-metadata-detector']['id'] | |||||
metadata_v1 = { | |||||
'developmentStatus': None, | |||||
'version': None, | |||||
'operatingSystem': None, | |||||
'description': None, | |||||
'keywords': None, | |||||
'issueTracker': None, | |||||
'name': None, | |||||
'author': None, | |||||
'relatedLink': None, | |||||
'url': None, | |||||
'license': None, | |||||
'maintainer': None, | |||||
'email': None, | |||||
'softwareRequirements': None, | |||||
'identifier': None | |||||
} | |||||
metadata_rev_v1 = { | |||||
'id': self.revision_id_2, | |||||
'translated_metadata': metadata_v1, | |||||
'indexer_configuration_id': tool_id, | |||||
} | |||||
metadata_origin_v1 = { | |||||
'origin_id': self.origin_id_1, | |||||
'metadata': metadata_v1.copy(), | |||||
'indexer_configuration_id': tool_id, | |||||
'from_revision': self.revision_id_2, | |||||
} | |||||
# given | |||||
self.storage.revision_metadata_add([metadata_rev_v1]) | |||||
self.storage.origin_intrinsic_metadata_add([metadata_origin_v1]) | |||||
# when | |||||
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( | |||||
[self.origin_id_1])) | |||||
# then | |||||
expected_metadata_v1 = [{ | |||||
'origin_id': self.origin_id_1, | |||||
'metadata': metadata_v1, | |||||
'tool': self.tools['swh-metadata-detector'], | |||||
'from_revision': self.revision_id_2, | |||||
}] | |||||
self.assertEqual(actual_metadata, expected_metadata_v1) | |||||
# given | |||||
metadata_v2 = metadata_v1.copy() | |||||
metadata_v2.update({ | |||||
'name': 'test_update_duplicated_metadata', | |||||
'author': 'MG', | |||||
}) | |||||
metadata_rev_v2 = metadata_rev_v1.copy() | |||||
metadata_origin_v2 = metadata_origin_v1.copy() | |||||
metadata_rev_v2['translated_metadata'] = metadata_v2 | |||||
metadata_origin_v2['metadata'] = metadata_v2 | |||||
self.storage.revision_metadata_add([metadata_rev_v2], | |||||
conflict_update=True) | |||||
self.storage.origin_intrinsic_metadata_add([metadata_origin_v2], | |||||
conflict_update=True) | |||||
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( | |||||
[self.origin_id_1])) | |||||
expected_metadata_v2 = [{ | |||||
'origin_id': self.origin_id_1, | |||||
'metadata': metadata_v2, | |||||
Done Inline Actionsv2 is not dropped here, i think the comment needs some update (or drop it? ;) ardumont: v2 is not dropped here, i think the comment needs some update (or drop it? ;) | |||||
'tool': self.tools['swh-metadata-detector'], | |||||
'from_revision': self.revision_id_2, | |||||
}] | |||||
# metadata did change as the v2 was used to overwrite v1 | |||||
self.assertEqual(actual_metadata, expected_metadata_v2) | |||||
def test_indexer_configuration_add(self): | def test_indexer_configuration_add(self): | ||||
tool = { | tool = { | ||||
'tool_name': 'some-unknown-tool', | 'tool_name': 'some-unknown-tool', | ||||
Not Done Inline ActionsI'm not sure why you revision_metadata_add in all tests since you don't check back what you inserted. ardumont: I'm not sure why you `revision_metadata_add` in all tests since you don't check back what you… | |||||
Done Inline ActionsRequired because of the FK. vlorentz: Required because of the FK. | |||||
'tool_version': 'some-version', | 'tool_version': 'some-version', | ||||
'tool_configuration': {"debian-package": "some-package"}, | 'tool_configuration': {"debian-package": "some-package"}, | ||||
} | } | ||||
actual_tool = self.storage.indexer_configuration_get(tool) | actual_tool = self.storage.indexer_configuration_get(tool) | ||||
self.assertIsNone(actual_tool) # does not exist | self.assertIsNone(actual_tool) # does not exist | ||||
# add it | # add it | ||||
▲ Show 20 Lines • Show All 101 Lines • Show Last 20 Lines |
type is no longer in the minimal set.
check all type properties in tests