Page MenuHomeSoftware Heritage
Differential D557 Diff 1790 swh/indexer/tests/storage/test_storage.py

Changeset View

Standalone View

View Options

swh/indexer/tests/storage/test_storage.py

Show All 35 Lines def setUp(self):
self.storage = get_indexer_storage(**self.storage_config) self.storage = get_indexer_storage(**self.storage_config)
self.sha1_1 = hash_to_bytes('34973274ccef6ab4dfaaf86599792fa9c3fe4689') self.sha1_1 = hash_to_bytes('34973274ccef6ab4dfaaf86599792fa9c3fe4689')
self.sha1_2 = hash_to_bytes('61c2b3a30496d329e21af70dd2d7e097046d07b7') self.sha1_2 = hash_to_bytes('61c2b3a30496d329e21af70dd2d7e097046d07b7')
self.revision_id_1 = hash_to_bytes( self.revision_id_1 = hash_to_bytes(
'7026b7c1a2af56521e951c01ed20f255fa054238') '7026b7c1a2af56521e951c01ed20f255fa054238')
self.revision_id_2 = hash_to_bytes( self.revision_id_2 = hash_to_bytes(
'7026b7c1a2af56521e9587659012345678904321') '7026b7c1a2af56521e9587659012345678904321')
self.origin_id_1 = 54974445
cur = self.test_db[self.TEST_DB_NAME].cursor cur = self.test_db[self.TEST_DB_NAME].cursor
tools = {} tools = {}
cur.execute(''' cur.execute('''
select tool_name, id, tool_version, tool_configuration select tool_name, id, tool_version, tool_configuration
from indexer_configuration from indexer_configuration
order by id''') order by id''')
for row in cur.fetchall(): for row in cur.fetchall():
▲ Show 20 Lines • Show All 1,072 Lines • ▼ Show 20 Lines def test_revision_metadata_missing(self):
'operatingSystem': None, 'operatingSystem': None,
'description': None, 'description': None,
'keywords': None, 'keywords': None,
'issueTracker': None, 'issueTracker': None,
'name': None, 'name': None,
'author': None, 'author': None,
'relatedLink': None, 'relatedLink': None,
'url': None, 'url': None,
'type': None,
'license': None, 'license': None,
'maintainer': None, 'maintainer': None,
'email': None, 'email': None,
'softwareRequirements': None, 'softwareRequirements': None,
'identifier': None 'identifier': None
}, },
'indexer_configuration_id': tool_id 'indexer_configuration_id': tool_id
}]) }])
Show All 17 Lines def test_revision_metadata_get(self):
'operatingSystem': None, 'operatingSystem': None,
'description': None, 'description': None,
'keywords': None, 'keywords': None,
'issueTracker': None, 'issueTracker': None,
'name': None, 'name': None,
'author': None, 'author': None,
'relatedLink': None, 'relatedLink': None,
'url': None, 'url': None,
'type': None,
'license': None, 'license': None,
'maintainer': None, 'maintainer': None,
'email': None, 'email': None,
'softwareRequirements': None, 'softwareRequirements': None,
'identifier': None 'identifier': None
}, },
'indexer_configuration_id': tool_id 'indexer_configuration_id': tool_id
} }
Show All 25 Lines def test_revision_metadata_add_drop_duplicate(self):
'operatingSystem': None, 'operatingSystem': None,
'description': None, 'description': None,
'keywords': None, 'keywords': None,
'issueTracker': None, 'issueTracker': None,
'name': None, 'name': None,
'author': None, 'author': None,
'relatedLink': None, 'relatedLink': None,
'url': None, 'url': None,
'type': None,
'license': None, 'license': None,
'maintainer': None, 'maintainer': None,
'email': None, 'email': None,
'softwareRequirements': None, 'softwareRequirements': None,
'identifier': None 'identifier': None
}, },
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines def test_revision_metadata_add_update_in_place_duplicate(self):
'operatingSystem': None, 'operatingSystem': None,
'description': None, 'description': None,
'keywords': None, 'keywords': None,
'issueTracker': None, 'issueTracker': None,
'name': None, 'name': None,
'author': None, 'author': None,
'relatedLink': None, 'relatedLink': None,
'url': None, 'url': None,
'type': None,
'license': None, 'license': None,
'maintainer': None, 'maintainer': None,
'email': None, 'email': None,
'softwareRequirements': None, 'softwareRequirements': None,
'identifier': None 'identifier': None
}, },
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
Show All 21 Lines def test_revision_metadata_add_update_in_place_duplicate(self):
'author': 'MG' 'author': 'MG'
}, },
}) })
self.storage.revision_metadata_add([metadata_v2], conflict_update=True) self.storage.revision_metadata_add([metadata_v2], conflict_update=True)
actual_metadata = list(self.storage.revision_metadata_get( actual_metadata = list(self.storage.revision_metadata_get(
[self.revision_id_2])) [self.revision_id_2]))
# language did not change as the v2 was dropped.
expected_metadata_v2 = [{ expected_metadata_v2 = [{
'id': self.revision_id_2, 'id': self.revision_id_2,
'translated_metadata': metadata_v2['translated_metadata'], 'translated_metadata': metadata_v2['translated_metadata'],
'tool': self.tools['swh-metadata-detector'] 'tool': self.tools['swh-metadata-detector']
}] }]
# metadata did change as the v2 was used to overwrite v1 # metadata did change as the v2 was used to overwrite v1
self.assertEqual(actual_metadata, expected_metadata_v2) self.assertEqual(actual_metadata, expected_metadata_v2)
def test_origin_intrinsic_metadata_get(self):
# given
tool_id = self.tools['swh-metadata-detector']['id']
metadata = {
'developmentStatus': None,
'version': None,
'operatingSystem': None,
'description': None,
'keywords': None,
'issueTracker': None,
'name': None,
'author': None,
'relatedLink': None,
'url': None,
moraneggUnsubmitted
Done
Inline Actions

type is no longer in the minimal set.
check all type properties in tests

moranegg: type is no longer in the minimal set. check all `type` properties in tests
'license': None,
'maintainer': None,
'email': None,
'softwareRequirements': None,
'identifier': None,
}
metadata_rev = {
'id': self.revision_id_2,
'translated_metadata': metadata,
'indexer_configuration_id': tool_id,
}
metadata_origin = {
'origin_id': self.origin_id_1,
'metadata': metadata,
'indexer_configuration_id': tool_id,
'from_revision': self.revision_id_2,
}
# when
self.storage.revision_metadata_add([metadata_rev])
self.storage.origin_intrinsic_metadata_add([metadata_origin])
# then
actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1, 42]))
expected_metadata = [{
'origin_id': self.origin_id_1,
'metadata': metadata,
'tool': self.tools['swh-metadata-detector'],
'from_revision': self.revision_id_2,
}]
self.assertEqual(actual_metadata, expected_metadata)
def test_origin_intrinsic_metadata_add_drop_duplicate(self):
# given
tool_id = self.tools['swh-metadata-detector']['id']
metadata_v1 = {
'developmentStatus': None,
'version': None,
'operatingSystem': None,
'description': None,
'keywords': None,
'issueTracker': None,
'name': None,
'author': None,
'relatedLink': None,
'url': None,
'license': None,
'maintainer': None,
'email': None,
'softwareRequirements': None,
'identifier': None
}
metadata_rev_v1 = {
'id': self.revision_id_1,
'translated_metadata': metadata_v1.copy(),
'indexer_configuration_id': tool_id,
}
metadata_origin_v1 = {
'origin_id': self.origin_id_1,
'metadata': metadata_v1.copy(),
'indexer_configuration_id': tool_id,
'from_revision': self.revision_id_1,
}
# given
self.storage.revision_metadata_add([metadata_rev_v1])
self.storage.origin_intrinsic_metadata_add([metadata_origin_v1])
# when
actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1, 42]))
expected_metadata_v1 = [{
'origin_id': self.origin_id_1,
'metadata': metadata_v1,
'tool': self.tools['swh-metadata-detector'],
'from_revision': self.revision_id_1,
}]
self.assertEqual(actual_metadata, expected_metadata_v1)
# given
metadata_v2 = metadata_v1.copy()
metadata_v2.update({
'name': 'test_metadata',
'author': 'MG',
})
metadata_rev_v2 = metadata_rev_v1.copy()
metadata_origin_v2 = metadata_origin_v1.copy()
metadata_rev_v2['translated_metadata'] = metadata_v2
metadata_origin_v2['translated_metadata'] = metadata_v2
self.storage.revision_metadata_add([metadata_rev_v2])
self.storage.origin_intrinsic_metadata_add([metadata_origin_v2])
# then
actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1]))
# metadata did not change as the v2 was dropped.
self.assertEqual(actual_metadata, expected_metadata_v1)
def test_origin_intrinsic_metadata_add_update_in_place_duplicate(self):
# given
tool_id = self.tools['swh-metadata-detector']['id']
metadata_v1 = {
'developmentStatus': None,
'version': None,
'operatingSystem': None,
'description': None,
'keywords': None,
'issueTracker': None,
'name': None,
'author': None,
'relatedLink': None,
'url': None,
'license': None,
'maintainer': None,
'email': None,
'softwareRequirements': None,
'identifier': None
}
metadata_rev_v1 = {
'id': self.revision_id_2,
'translated_metadata': metadata_v1,
'indexer_configuration_id': tool_id,
}
metadata_origin_v1 = {
'origin_id': self.origin_id_1,
'metadata': metadata_v1.copy(),
'indexer_configuration_id': tool_id,
'from_revision': self.revision_id_2,
}
# given
self.storage.revision_metadata_add([metadata_rev_v1])
self.storage.origin_intrinsic_metadata_add([metadata_origin_v1])
# when
actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1]))
# then
expected_metadata_v1 = [{
'origin_id': self.origin_id_1,
'metadata': metadata_v1,
'tool': self.tools['swh-metadata-detector'],
'from_revision': self.revision_id_2,
}]
self.assertEqual(actual_metadata, expected_metadata_v1)
# given
metadata_v2 = metadata_v1.copy()
metadata_v2.update({
'name': 'test_update_duplicated_metadata',
'author': 'MG',
})
metadata_rev_v2 = metadata_rev_v1.copy()
metadata_origin_v2 = metadata_origin_v1.copy()
metadata_rev_v2['translated_metadata'] = metadata_v2
metadata_origin_v2['metadata'] = metadata_v2
self.storage.revision_metadata_add([metadata_rev_v2],
conflict_update=True)
self.storage.origin_intrinsic_metadata_add([metadata_origin_v2],
conflict_update=True)
actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1]))
expected_metadata_v2 = [{
'origin_id': self.origin_id_1,
'metadata': metadata_v2,
ardumontUnsubmitted
Done
Inline Actions

v2 is not dropped here, i think the comment needs some update (or drop it? ;)

ardumont: v2 is not dropped here, i think the comment needs some update (or drop it? ;)
'tool': self.tools['swh-metadata-detector'],
'from_revision': self.revision_id_2,
}]
# metadata did change as the v2 was used to overwrite v1
self.assertEqual(actual_metadata, expected_metadata_v2)
def test_indexer_configuration_add(self): def test_indexer_configuration_add(self):
tool = { tool = {
'tool_name': 'some-unknown-tool', 'tool_name': 'some-unknown-tool',
ardumontUnsubmitted
Not Done
Inline Actions

I'm not sure why you revision_metadata_add in all tests since you don't check back what you inserted.
I might be needed a second pass ;)

ardumont: I'm not sure why you `revision_metadata_add` in all tests since you don't check back what you…
vlorentzAuthorUnsubmitted
Done
Inline Actions

Required because of the FK.

vlorentz: Required because of the FK.
'tool_version': 'some-version', 'tool_version': 'some-version',
'tool_configuration': {"debian-package": "some-package"}, 'tool_configuration': {"debian-package": "some-package"},
} }
actual_tool = self.storage.indexer_configuration_get(tool) actual_tool = self.storage.indexer_configuration_get(tool)
self.assertIsNone(actual_tool) # does not exist self.assertIsNone(actual_tool) # does not exist
# add it # add it
▲ Show 20 Lines • Show All 101 Lines • Show Last 20 Lines
About · Developer information · Legal