Page MenuHomeSoftware Heritage
Differential D1564 Diff 5462 swh/indexer/tests/storage/test_storage.py

Changeset View

Standalone View

View Options

swh/indexer/tests/storage/test_storage.py

Show First 20 Lines • Show All 997 Lines • ▼ Show 20 Lines def test_origin_intrinsic_metadata_get(self):
metadata_rev = { metadata_rev = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'metadata': metadata, 'metadata': metadata,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata_origin = { metadata_origin = {
'id': self.origin_id_1, 'id': self.origin_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata, 'metadata': metadata,
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
# when # when
self.storage.revision_intrinsic_metadata_add([metadata_rev]) self.storage.revision_intrinsic_metadata_add([metadata_rev])
self.storage.origin_intrinsic_metadata_add([metadata_origin]) self.storage.origin_intrinsic_metadata_add([metadata_origin])
# then # then
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1, 42])) [self.origin_id_1, 42]))
expected_metadata = [{ expected_metadata = [{
'id': self.origin_id_1, 'id': self.origin_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata, 'metadata': metadata,
'tool': self.tools['swh-metadata-detector'], 'tool': self.tools['swh-metadata-detector'],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
}] }]
self.assertEqual(actual_metadata, expected_metadata) self.assertEqual(actual_metadata, expected_metadata)
def test_origin_intrinsic_metadata_delete(self): def test_origin_intrinsic_metadata_delete(self):
# given # given
tool_id = self.tools['swh-metadata-detector']['id'] tool_id = self.tools['swh-metadata-detector']['id']
metadata = { metadata = {
'version': None, 'version': None,
'name': None, 'name': None,
} }
metadata_rev = { metadata_rev = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'metadata': metadata, 'metadata': metadata,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata_origin = { metadata_origin = {
'id': self.origin_id_1, 'id': self.origin_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata, 'metadata': metadata,
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
metadata_origin2 = metadata_origin.copy() metadata_origin2 = metadata_origin.copy()
metadata_origin2['id'] = self.origin_id_2 metadata_origin2['id'] = self.origin_id_2
Show All 30 Lines def test_origin_intrinsic_metadata_add_drop_duplicate(self):
tool_id = self.tools['swh-metadata-detector']['id'] tool_id = self.tools['swh-metadata-detector']['id']
metadata_v1 = { metadata_v1 = {
'version': None, 'version': None,
'name': None, 'name': None,
} }
metadata_rev_v1 = { metadata_rev_v1 = {
'id': self.revision_id_1, 'id': self.revision_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata_v1.copy(), 'metadata': metadata_v1.copy(),
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata_origin_v1 = { metadata_origin_v1 = {
'id': self.origin_id_1, 'id': self.origin_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata_v1.copy(), 'metadata': metadata_v1.copy(),
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'mappings': [], 'mappings': [],
'from_revision': self.revision_id_1, 'from_revision': self.revision_id_1,
} }
# given # given
self.storage.revision_intrinsic_metadata_add([metadata_rev_v1]) self.storage.revision_intrinsic_metadata_add([metadata_rev_v1])
self.storage.origin_intrinsic_metadata_add([metadata_origin_v1]) self.storage.origin_intrinsic_metadata_add([metadata_origin_v1])
# when # when
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1, 42])) [self.origin_id_1, 42]))
expected_metadata_v1 = [{ expected_metadata_v1 = [{
'id': self.origin_id_1, 'id': self.origin_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata_v1, 'metadata': metadata_v1,
'tool': self.tools['swh-metadata-detector'], 'tool': self.tools['swh-metadata-detector'],
'from_revision': self.revision_id_1, 'from_revision': self.revision_id_1,
'mappings': [], 'mappings': [],
}] }]
self.assertEqual(actual_metadata, expected_metadata_v1) self.assertEqual(actual_metadata, expected_metadata_v1)
Show All 29 Lines def test_origin_intrinsic_metadata_add_update_in_place_duplicate(self):
metadata_rev_v1 = { metadata_rev_v1 = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'metadata': metadata_v1, 'metadata': metadata_v1,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata_origin_v1 = { metadata_origin_v1 = {
'id': self.origin_id_1, 'id': self.origin_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata_v1.copy(), 'metadata': metadata_v1.copy(),
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'mappings': [], 'mappings': [],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
# given # given
self.storage.revision_intrinsic_metadata_add([metadata_rev_v1]) self.storage.revision_intrinsic_metadata_add([metadata_rev_v1])
self.storage.origin_intrinsic_metadata_add([metadata_origin_v1]) self.storage.origin_intrinsic_metadata_add([metadata_origin_v1])
# when # when
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1])) [self.origin_id_1]))
# then # then
expected_metadata_v1 = [{ expected_metadata_v1 = [{
'id': self.origin_id_1, 'id': self.origin_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata_v1, 'metadata': metadata_v1,
'tool': self.tools['swh-metadata-detector'], 'tool': self.tools['swh-metadata-detector'],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
'mappings': [], 'mappings': [],
}] }]
self.assertEqual(actual_metadata, expected_metadata_v1) self.assertEqual(actual_metadata, expected_metadata_v1)
# given # given
Show All 12 Lines def test_origin_intrinsic_metadata_add_update_in_place_duplicate(self):
self.storage.origin_intrinsic_metadata_add( self.storage.origin_intrinsic_metadata_add(
[metadata_origin_v2], conflict_update=True) [metadata_origin_v2], conflict_update=True)
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1])) [self.origin_id_1]))
expected_metadata_v2 = [{ expected_metadata_v2 = [{
'id': self.origin_id_1, 'id': self.origin_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata_v2, 'metadata': metadata_v2,
'tool': self.tools['swh-metadata-detector'], 'tool': self.tools['swh-metadata-detector'],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
'mappings': [], 'mappings': [],
}] }]
# metadata did change as the v2 was used to overwrite v1 # metadata did change as the v2 was used to overwrite v1
self.assertEqual(actual_metadata, expected_metadata_v2) self.assertEqual(actual_metadata, expected_metadata_v2)
def test_origin_intrinsic_metadata_add__update_in_place_deadlock(self): def test_origin_intrinsic_metadata_add__update_in_place_deadlock(self):
# given # given
tool_id = self.tools['swh-metadata-detector']['id'] tool_id = self.tools['swh-metadata-detector']['id']
ids = list(range(1000)) ids = list(range(10))
example_data1 = { example_data1 = {
'metadata': { 'metadata': {
'version': None, 'version': None,
'name': None, 'name': None,
}, },
'mappings': [], 'mappings': [],
} }
Show All 13 Lines def test_origin_intrinsic_metadata_add__update_in_place_deadlock(self):
}, },
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
data_v1 = [ data_v1 = [
{ {
'id': id_, 'id': id_,
'origin_url': 'file:///tmp/origin%d' % id_,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
**example_data1, **example_data1,
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
for id_ in ids for id_ in ids
] ]
data_v2 = [ data_v2 = [
{ {
'id': id_, 'id': id_,
'origin_url': 'file:///tmp/origin%d' % id_,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
**example_data2, **example_data2,
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
for id_ in ids for id_ in ids
] ]
# Remove one item from each, so that both queries have to succeed for # Remove one item from each, so that both queries have to succeed for
# all items to be in the DB. # all items to be in the DB.
data_v2a = data_v2[1:] data_v2a = data_v2[1:]
data_v2b = list(reversed(data_v2[0:-1])) data_v2b = list(reversed(data_v2[0:-1]))
# given # given
self.storage.revision_intrinsic_metadata_add([metadata_rev_v1]) self.storage.revision_intrinsic_metadata_add([metadata_rev_v1])
self.storage.origin_intrinsic_metadata_add(data_v1) self.storage.origin_intrinsic_metadata_add(data_v1)
# when # when
actual_data = list(self.storage.origin_intrinsic_metadata_get(ids)) actual_data = list(self.storage.origin_intrinsic_metadata_get(ids))
expected_data_v1 = [ expected_data_v1 = [
{ {
'id': id_, 'id': id_,
'origin_url': 'file:///tmp/origin%d' % id_,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
**example_data1, **example_data1,
'tool': self.tools['swh-metadata-detector'], 'tool': self.tools['swh-metadata-detector'],
} }
for id_ in ids for id_ in ids
] ]
# then # then
Show All 16 Lines def test_origin_intrinsic_metadata_add__update_in_place_deadlock(self):
t1.join() t1.join()
t2.join() t2.join()
actual_data = list(self.storage.origin_intrinsic_metadata_get(ids)) actual_data = list(self.storage.origin_intrinsic_metadata_get(ids))
expected_data_v2 = [ expected_data_v2 = [
{ {
'id': id_, 'id': id_,
'origin_url': 'file:///tmp/origin%d' % id_,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
**example_data2, **example_data2,
'tool': self.tools['swh-metadata-detector'], 'tool': self.tools['swh-metadata-detector'],
} }
for id_ in ids for id_ in ids
] ]
self.maxDiff = None self.maxDiff = None
Show All 10 Lines def test_origin_intrinsic_metadata_add__duplicate_twice(self):
metadata_rev = { metadata_rev = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'metadata': metadata, 'metadata': metadata,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata_origin = { metadata_origin = {
'id': self.origin_id_1, 'id': self.origin_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata, 'metadata': metadata,
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
# when # when
self.storage.revision_intrinsic_metadata_add([metadata_rev]) self.storage.revision_intrinsic_metadata_add([metadata_rev])
Show All 12 Lines def test_origin_intrinsic_metadata_search_fulltext(self):
metadata1_rev = { metadata1_rev = {
'id': self.revision_id_1, 'id': self.revision_id_1,
'metadata': metadata1, 'metadata': metadata1,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata1_origin = { metadata1_origin = {
'id': self.origin_id_1, 'id': self.origin_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata1, 'metadata': metadata1,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'from_revision': self.revision_id_1, 'from_revision': self.revision_id_1,
} }
metadata2 = { metadata2 = {
'author': 'Jane Doe', 'author': 'Jane Doe',
} }
metadata2_rev = { metadata2_rev = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'origin_url': 'file:///dev/zero',
'metadata': metadata2, 'metadata': metadata2,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata2_origin = { metadata2_origin = {
'id': self.origin_id_2, 'id': self.origin_id_2,
'origin_url': 'file:///dev/zero',
'metadata': metadata2, 'metadata': metadata2,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
# when # when
self.storage.revision_intrinsic_metadata_add([metadata1_rev]) self.storage.revision_intrinsic_metadata_add([metadata1_rev])
Show All 34 Lines def test_origin_intrinsic_metadata_search_fulltext_rank(self):
metadata1_rev = { metadata1_rev = {
'id': self.revision_id_1, 'id': self.revision_id_1,
'metadata': metadata1, 'metadata': metadata1,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata1_origin = { metadata1_origin = {
'id': self.origin_id_1, 'id': self.origin_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata1, 'metadata': metadata1,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'from_revision': self.revision_id_1, 'from_revision': self.revision_id_1,
} }
metadata2 = { metadata2 = {
'author': [ 'author': [
'Random Person', 'Random Person',
'Jane Doe', 'Jane Doe',
] ]
} }
metadata2_rev = { metadata2_rev = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'metadata': metadata2, 'metadata': metadata2,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata2_origin = { metadata2_origin = {
'id': self.origin_id_2, 'id': self.origin_id_2,
'origin_url': 'file:///dev/zero',
'metadata': metadata2, 'metadata': metadata2,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
# when # when
self.storage.revision_intrinsic_metadata_add([metadata1_rev]) self.storage.revision_intrinsic_metadata_add([metadata1_rev])
Show All 30 Lines def _fill_origin_intrinsic_metadata(self):
metadata1_rev = { metadata1_rev = {
'id': self.revision_id_1, 'id': self.revision_id_1,
'metadata': metadata1, 'metadata': metadata1,
'mappings': ['npm'], 'mappings': ['npm'],
'indexer_configuration_id': tool1_id, 'indexer_configuration_id': tool1_id,
} }
metadata1_origin = { metadata1_origin = {
'id': self.origin_id_1, 'id': self.origin_id_1,
'origin_url': 'file:///dev/zero',
'metadata': metadata1, 'metadata': metadata1,
'mappings': ['npm'], 'mappings': ['npm'],
'indexer_configuration_id': tool1_id, 'indexer_configuration_id': tool1_id,
'from_revision': self.revision_id_1, 'from_revision': self.revision_id_1,
} }
metadata2 = { metadata2 = {
'@context': 'foo', '@context': 'foo',
'author': 'Jane Doe', 'author': 'Jane Doe',
} }
metadata2_rev = { metadata2_rev = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'metadata': metadata2, 'metadata': metadata2,
'mappings': ['npm', 'gemspec'], 'mappings': ['npm', 'gemspec'],
'indexer_configuration_id': tool2_id, 'indexer_configuration_id': tool2_id,
} }
metadata2_origin = { metadata2_origin = {
'id': self.origin_id_2, 'id': self.origin_id_2,
'origin_url': 'file:///dev/zero',
'metadata': metadata2, 'metadata': metadata2,
'mappings': ['npm', 'gemspec'], 'mappings': ['npm', 'gemspec'],
'indexer_configuration_id': tool2_id, 'indexer_configuration_id': tool2_id,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
metadata3 = { metadata3 = {
'@context': 'foo', '@context': 'foo',
} }
metadata3_rev = { metadata3_rev = {
'id': self.revision_id_3, 'id': self.revision_id_3,
'metadata': metadata3, 'metadata': metadata3,
'mappings': ['npm', 'gemspec'], 'mappings': ['npm', 'gemspec'],
'indexer_configuration_id': tool2_id, 'indexer_configuration_id': tool2_id,
} }
metadata3_origin = { metadata3_origin = {
'id': self.origin_id_3, 'id': self.origin_id_3,
'origin_url': 'file:///dev/zero',
'metadata': metadata3, 'metadata': metadata3,
'mappings': ['pkg-info'], 'mappings': ['pkg-info'],
'indexer_configuration_id': tool2_id, 'indexer_configuration_id': tool2_id,
'from_revision': self.revision_id_3, 'from_revision': self.revision_id_3,
} }
self.storage.revision_intrinsic_metadata_add([metadata1_rev]) self.storage.revision_intrinsic_metadata_add([metadata1_rev])
self.storage.origin_intrinsic_metadata_add([metadata1_origin]) self.storage.origin_intrinsic_metadata_add([metadata1_origin])
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines def test_origin_intrinsic_metadata_search_by_producer(self):
[self.origin_id_2, self.origin_id_3]) [self.origin_id_2, self.origin_id_3])
self.assertCountEqual( self.assertCountEqual(
endpoint(tool_ids=[tool1['id'], tool2['id']], ids_only=True), endpoint(tool_ids=[tool1['id'], tool2['id']], ids_only=True),
[self.origin_id_1, self.origin_id_2, self.origin_id_3]) [self.origin_id_1, self.origin_id_2, self.origin_id_3])
# test ids_only=False # test ids_only=False
self.assertEqual(list(endpoint(mappings=['gemspec'])), [{ self.assertEqual(list(endpoint(mappings=['gemspec'])), [{
'id': self.origin_id_2, 'id': self.origin_id_2,
'origin_url': 'file:///dev/zero',
'metadata': { 'metadata': {
'@context': 'foo', '@context': 'foo',
'author': 'Jane Doe', 'author': 'Jane Doe',
}, },
'mappings': ['npm', 'gemspec'], 'mappings': ['npm', 'gemspec'],
'tool': tool2, 'tool': tool2,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
}]) }])
▲ Show 20 Lines • Show All 356 Lines • Show Last 20 Lines
About · Developer information · Legal