Page MenuHomeSoftware Heritage
Differential D1226 Diff 3909 swh/indexer/tests/storage/test_storage.py

Changeset View

Standalone View

View Options

swh/indexer/tests/storage/test_storage.py

Show First 20 Lines • Show All 925 Lines • ▼ Show 20 Lines (
test_content_metadata_add__duplicate_twice, test_content_metadata_add__duplicate_twice,
test_content_metadata_get, test_content_metadata_get,
_, # test_content_metadata_delete, _, # test_content_metadata_delete,
_, # test_content_metadata_delete_nonexisting, _, # test_content_metadata_delete_nonexisting,
) = gen_generic_endpoint_tests( ) = gen_generic_endpoint_tests(
endpoint_type='content_metadata', endpoint_type='content_metadata',
tool_name='swh-metadata-detector', tool_name='swh-metadata-detector',
example_data1={ example_data1={
'translated_metadata': { 'metadata': {
'other': {}, 'other': {},
'codeRepository': { 'codeRepository': {
'type': 'git', 'type': 'git',
'url': 'https://github.com/moranegg/metadata_test' 'url': 'https://github.com/moranegg/metadata_test'
}, },
'description': 'Simple package.json test for indexer', 'description': 'Simple package.json test for indexer',
'name': 'test_metadata', 'name': 'test_metadata',
'version': '0.0.1' 'version': '0.0.1'
}, },
}, },
example_data2={ example_data2={
'translated_metadata': { 'metadata': {
'other': {}, 'other': {},
'name': 'test_metadata', 'name': 'test_metadata',
'version': '0.0.1' 'version': '0.0.1'
}, },
}, },
) )
# revision_metadata tests # revision_intrinsic_metadata tests
( (
test_revision_metadata_missing, test_revision_intrinsic_metadata_missing,
test_revision_metadata_add__drop_duplicate, test_revision_intrinsic_metadata_add__drop_duplicate,
test_revision_metadata_add__update_in_place_duplicate, test_revision_intrinsic_metadata_add__update_in_place_duplicate,
test_revision_metadata_add__update_in_place_deadlock, test_revision_intrinsic_metadata_add__update_in_place_deadlock,
test_revision_metadata_add__duplicate_twice, test_revision_intrinsic_metadata_add__duplicate_twice,
test_revision_metadata_get, test_revision_intrinsic_metadata_get,
test_revision_metadata_delete, test_revision_intrinsic_metadata_delete,
test_revision_metadata_delete_nonexisting, test_revision_intrinsic_metadata_delete_nonexisting,
) = gen_generic_endpoint_tests( ) = gen_generic_endpoint_tests(
endpoint_type='revision_metadata', endpoint_type='revision_intrinsic_metadata',
tool_name='swh-metadata-detector', tool_name='swh-metadata-detector',
example_data1={ example_data1={
'translated_metadata': { 'metadata': {
'other': {}, 'other': {},
'codeRepository': { 'codeRepository': {
'type': 'git', 'type': 'git',
'url': 'https://github.com/moranegg/metadata_test' 'url': 'https://github.com/moranegg/metadata_test'
}, },
'description': 'Simple package.json test for indexer', 'description': 'Simple package.json test for indexer',
'name': 'test_metadata', 'name': 'test_metadata',
'version': '0.0.1' 'version': '0.0.1'
}, },
'mappings': ['mapping1'], 'mappings': ['mapping1'],
}, },
example_data2={ example_data2={
'translated_metadata': { 'metadata': {
'other': {}, 'other': {},
'name': 'test_metadata', 'name': 'test_metadata',
'version': '0.0.1' 'version': '0.0.1'
}, },
'mappings': ['mapping2'], 'mappings': ['mapping2'],
}, },
) )
def test_origin_intrinsic_metadata_get(self): def test_origin_intrinsic_metadata_get(self):
# given # given
tool_id = self.tools['swh-metadata-detector']['id'] tool_id = self.tools['swh-metadata-detector']['id']
metadata = { metadata = {
'version': None, 'version': None,
'name': None, 'name': None,
} }
metadata_rev = { metadata_rev = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'translated_metadata': metadata, 'metadata': metadata,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata_origin = { metadata_origin = {
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'metadata': metadata, 'metadata': metadata,
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
# when # when
self.storage.revision_metadata_add([metadata_rev]) self.storage.revision_intrinsic_metadata_add([metadata_rev])
self.storage.origin_intrinsic_metadata_add([metadata_origin]) self.storage.origin_intrinsic_metadata_add([metadata_origin])
# then # then
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1, 42])) [self.origin_id_1, 42]))
expected_metadata = [{ expected_metadata = [{
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'metadata': metadata, 'metadata': metadata,
'tool': self.tools['swh-metadata-detector'], 'tool': self.tools['swh-metadata-detector'],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
}] }]
self.assertEqual(actual_metadata, expected_metadata) self.assertEqual(actual_metadata, expected_metadata)
def test_origin_intrinsic_metadata_delete(self): def test_origin_intrinsic_metadata_delete(self):
# given # given
tool_id = self.tools['swh-metadata-detector']['id'] tool_id = self.tools['swh-metadata-detector']['id']
metadata = { metadata = {
'version': None, 'version': None,
'name': None, 'name': None,
} }
metadata_rev = { metadata_rev = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'translated_metadata': metadata, 'metadata': metadata,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata_origin = { metadata_origin = {
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'metadata': metadata, 'metadata': metadata,
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
metadata_origin2 = metadata_origin.copy() metadata_origin2 = metadata_origin.copy()
metadata_origin2['origin_id'] = self.origin_id_2 metadata_origin2['id'] = self.origin_id_2
# when # when
self.storage.revision_metadata_add([metadata_rev]) self.storage.revision_intrinsic_metadata_add([metadata_rev])
self.storage.origin_intrinsic_metadata_add([ self.storage.origin_intrinsic_metadata_add([
metadata_origin, metadata_origin2]) metadata_origin, metadata_origin2])
self.storage.origin_intrinsic_metadata_delete([ self.storage.origin_intrinsic_metadata_delete([
{ {
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'indexer_configuration_id': tool_id 'indexer_configuration_id': tool_id
} }
]) ])
# then # then
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1, self.origin_id_2, 42])) [self.origin_id_1, self.origin_id_2, 42]))
for item in actual_metadata: for item in actual_metadata:
item['indexer_configuration_id'] = item.pop('tool')['id'] item['indexer_configuration_id'] = item.pop('tool')['id']
self.assertEqual(actual_metadata, [metadata_origin2]) self.assertEqual(actual_metadata, [metadata_origin2])
def test_origin_intrinsic_metadata_delete_nonexisting(self): def test_origin_intrinsic_metadata_delete_nonexisting(self):
tool_id = self.tools['swh-metadata-detector']['id'] tool_id = self.tools['swh-metadata-detector']['id']
self.storage.origin_intrinsic_metadata_delete([ self.storage.origin_intrinsic_metadata_delete([
{ {
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'indexer_configuration_id': tool_id 'indexer_configuration_id': tool_id
} }
]) ])
def test_origin_intrinsic_metadata_add_drop_duplicate(self): def test_origin_intrinsic_metadata_add_drop_duplicate(self):
# given # given
tool_id = self.tools['swh-metadata-detector']['id'] tool_id = self.tools['swh-metadata-detector']['id']
metadata_v1 = { metadata_v1 = {
'version': None, 'version': None,
'name': None, 'name': None,
} }
metadata_rev_v1 = { metadata_rev_v1 = {
'id': self.revision_id_1, 'id': self.revision_id_1,
'translated_metadata': metadata_v1.copy(), 'metadata': metadata_v1.copy(),
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata_origin_v1 = { metadata_origin_v1 = {
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'metadata': metadata_v1.copy(), 'metadata': metadata_v1.copy(),
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'mappings': [], 'mappings': [],
'from_revision': self.revision_id_1, 'from_revision': self.revision_id_1,
} }
# given # given
self.storage.revision_metadata_add([metadata_rev_v1]) self.storage.revision_intrinsic_metadata_add([metadata_rev_v1])
self.storage.origin_intrinsic_metadata_add([metadata_origin_v1]) self.storage.origin_intrinsic_metadata_add([metadata_origin_v1])
# when # when
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1, 42])) [self.origin_id_1, 42]))
expected_metadata_v1 = [{ expected_metadata_v1 = [{
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'metadata': metadata_v1, 'metadata': metadata_v1,
'tool': self.tools['swh-metadata-detector'], 'tool': self.tools['swh-metadata-detector'],
'from_revision': self.revision_id_1, 'from_revision': self.revision_id_1,
'mappings': [], 'mappings': [],
}] }]
self.assertEqual(actual_metadata, expected_metadata_v1) self.assertEqual(actual_metadata, expected_metadata_v1)
# given # given
metadata_v2 = metadata_v1.copy() metadata_v2 = metadata_v1.copy()
metadata_v2.update({ metadata_v2.update({
'name': 'test_metadata', 'name': 'test_metadata',
'author': 'MG', 'author': 'MG',
}) })
metadata_rev_v2 = metadata_rev_v1.copy() metadata_rev_v2 = metadata_rev_v1.copy()
metadata_origin_v2 = metadata_origin_v1.copy() metadata_origin_v2 = metadata_origin_v1.copy()
metadata_rev_v2['translated_metadata'] = metadata_v2 metadata_rev_v2['metadata'] = metadata_v2
metadata_origin_v2['translated_metadata'] = metadata_v2 metadata_origin_v2['metadata'] = metadata_v2
self.storage.revision_metadata_add([metadata_rev_v2]) self.storage.revision_intrinsic_metadata_add([metadata_rev_v2])
self.storage.origin_intrinsic_metadata_add([metadata_origin_v2]) self.storage.origin_intrinsic_metadata_add([metadata_origin_v2])
# then # then
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1])) [self.origin_id_1]))
# metadata did not change as the v2 was dropped. # metadata did not change as the v2 was dropped.
self.assertEqual(actual_metadata, expected_metadata_v1) self.assertEqual(actual_metadata, expected_metadata_v1)
def test_origin_intrinsic_metadata_add_update_in_place_duplicate(self): def test_origin_intrinsic_metadata_add_update_in_place_duplicate(self):
# given # given
tool_id = self.tools['swh-metadata-detector']['id'] tool_id = self.tools['swh-metadata-detector']['id']
metadata_v1 = { metadata_v1 = {
'version': None, 'version': None,
'name': None, 'name': None,
} }
metadata_rev_v1 = { metadata_rev_v1 = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'translated_metadata': metadata_v1, 'metadata': metadata_v1,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata_origin_v1 = { metadata_origin_v1 = {
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'metadata': metadata_v1.copy(), 'metadata': metadata_v1.copy(),
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'mappings': [], 'mappings': [],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
# given # given
self.storage.revision_metadata_add([metadata_rev_v1]) self.storage.revision_intrinsic_metadata_add([metadata_rev_v1])
self.storage.origin_intrinsic_metadata_add([metadata_origin_v1]) self.storage.origin_intrinsic_metadata_add([metadata_origin_v1])
# when # when
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1])) [self.origin_id_1]))
# then # then
expected_metadata_v1 = [{ expected_metadata_v1 = [{
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'metadata': metadata_v1, 'metadata': metadata_v1,
'tool': self.tools['swh-metadata-detector'], 'tool': self.tools['swh-metadata-detector'],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
'mappings': [], 'mappings': [],
}] }]
self.assertEqual(actual_metadata, expected_metadata_v1) self.assertEqual(actual_metadata, expected_metadata_v1)
# given # given
metadata_v2 = metadata_v1.copy() metadata_v2 = metadata_v1.copy()
metadata_v2.update({ metadata_v2.update({
'name': 'test_update_duplicated_metadata', 'name': 'test_update_duplicated_metadata',
'author': 'MG', 'author': 'MG',
}) })
metadata_rev_v2 = metadata_rev_v1.copy() metadata_rev_v2 = metadata_rev_v1.copy()
metadata_origin_v2 = metadata_origin_v1.copy() metadata_origin_v2 = metadata_origin_v1.copy()
metadata_rev_v2['translated_metadata'] = metadata_v2 metadata_rev_v2['metadata'] = metadata_v2
metadata_origin_v2['metadata'] = metadata_v2 metadata_origin_v2['metadata'] = metadata_v2
self.storage.revision_metadata_add([metadata_rev_v2], self.storage.revision_intrinsic_metadata_add(
conflict_update=True) [metadata_rev_v2], conflict_update=True)
self.storage.origin_intrinsic_metadata_add([metadata_origin_v2], self.storage.origin_intrinsic_metadata_add(
conflict_update=True) [metadata_origin_v2], conflict_update=True)
actual_metadata = list(self.storage.origin_intrinsic_metadata_get( actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
[self.origin_id_1])) [self.origin_id_1]))
expected_metadata_v2 = [{ expected_metadata_v2 = [{
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'metadata': metadata_v2, 'metadata': metadata_v2,
'tool': self.tools['swh-metadata-detector'], 'tool': self.tools['swh-metadata-detector'],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
'mappings': [], 'mappings': [],
}] }]
# metadata did change as the v2 was used to overwrite v1 # metadata did change as the v2 was used to overwrite v1
self.assertEqual(actual_metadata, expected_metadata_v2) self.assertEqual(actual_metadata, expected_metadata_v2)
Show All 16 Lines def test_origin_intrinsic_metadata_add__update_in_place_deadlock(self):
'version': 'v1.1.1', 'version': 'v1.1.1',
'name': 'foo', 'name': 'foo',
}, },
'mappings': [], 'mappings': [],
} }
metadata_rev_v1 = { metadata_rev_v1 = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'translated_metadata': { 'metadata': {
'version': None, 'version': None,
'name': None, 'name': None,
}, },
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
data_v1 = [ data_v1 = [
{ {
'origin_id': id_, 'id': id_,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
**example_data1, **example_data1,
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
for id_ in ids for id_ in ids
] ]
data_v2 = [ data_v2 = [
{ {
'origin_id': id_, 'id': id_,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
**example_data2, **example_data2,
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
for id_ in ids for id_ in ids
] ]
# Remove one item from each, so that both queries have to succeed for # Remove one item from each, so that both queries have to succeed for
# all items to be in the DB. # all items to be in the DB.
data_v2a = data_v2[1:] data_v2a = data_v2[1:]
data_v2b = list(reversed(data_v2[0:-1])) data_v2b = list(reversed(data_v2[0:-1]))
# given # given
self.storage.revision_metadata_add([metadata_rev_v1]) self.storage.revision_intrinsic_metadata_add([metadata_rev_v1])
self.storage.origin_intrinsic_metadata_add(data_v1) self.storage.origin_intrinsic_metadata_add(data_v1)
# when # when
actual_data = list(self.storage.origin_intrinsic_metadata_get(ids)) actual_data = list(self.storage.origin_intrinsic_metadata_get(ids))
expected_data_v1 = [ expected_data_v1 = [
{ {
'origin_id': id_, 'id': id_,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
**example_data1, **example_data1,
'tool': self.tools['swh-metadata-detector'], 'tool': self.tools['swh-metadata-detector'],
} }
for id_ in ids for id_ in ids
] ]
# then # then
Show All 15 Lines def test_origin_intrinsic_metadata_add__update_in_place_deadlock(self):
t1.join() t1.join()
t2.join() t2.join()
actual_data = list(self.storage.origin_intrinsic_metadata_get(ids)) actual_data = list(self.storage.origin_intrinsic_metadata_get(ids))
expected_data_v2 = [ expected_data_v2 = [
{ {
'origin_id': id_, 'id': id_,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
**example_data2, **example_data2,
'tool': self.tools['swh-metadata-detector'], 'tool': self.tools['swh-metadata-detector'],
} }
for id_ in ids for id_ in ids
] ]
self.maxDiff = None self.maxDiff = None
self.assertCountEqual(actual_data, expected_data_v2) self.assertCountEqual(actual_data, expected_data_v2)
def test_origin_intrinsic_metadata_add__duplicate_twice(self): def test_origin_intrinsic_metadata_add__duplicate_twice(self):
# given # given
tool_id = self.tools['swh-metadata-detector']['id'] tool_id = self.tools['swh-metadata-detector']['id']
metadata = { metadata = {
'developmentStatus': None, 'developmentStatus': None,
'name': None, 'name': None,
} }
metadata_rev = { metadata_rev = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'translated_metadata': metadata, 'metadata': metadata,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata_origin = { metadata_origin = {
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'metadata': metadata, 'metadata': metadata,
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'mappings': ['mapping1'], 'mappings': ['mapping1'],
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
# when # when
self.storage.revision_metadata_add([metadata_rev]) self.storage.revision_intrinsic_metadata_add([metadata_rev])
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
self.storage.origin_intrinsic_metadata_add([ self.storage.origin_intrinsic_metadata_add([
metadata_origin, metadata_origin]) metadata_origin, metadata_origin])
def test_origin_intrinsic_metadata_search_fulltext(self): def test_origin_intrinsic_metadata_search_fulltext(self):
# given # given
tool_id = self.tools['swh-metadata-detector']['id'] tool_id = self.tools['swh-metadata-detector']['id']
metadata1 = { metadata1 = {
'author': 'John Doe', 'author': 'John Doe',
} }
metadata1_rev = { metadata1_rev = {
'id': self.revision_id_1, 'id': self.revision_id_1,
'translated_metadata': metadata1, 'metadata': metadata1,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata1_origin = { metadata1_origin = {
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'metadata': metadata1, 'metadata': metadata1,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'from_revision': self.revision_id_1, 'from_revision': self.revision_id_1,
} }
metadata2 = { metadata2 = {
'author': 'Jane Doe', 'author': 'Jane Doe',
} }
metadata2_rev = { metadata2_rev = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'translated_metadata': metadata2, 'metadata': metadata2,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata2_origin = { metadata2_origin = {
'origin_id': self.origin_id_2, 'id': self.origin_id_2,
'metadata': metadata2, 'metadata': metadata2,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
# when # when
self.storage.revision_metadata_add([metadata1_rev]) self.storage.revision_intrinsic_metadata_add([metadata1_rev])
self.storage.origin_intrinsic_metadata_add([metadata1_origin]) self.storage.origin_intrinsic_metadata_add([metadata1_origin])
self.storage.revision_metadata_add([metadata2_rev]) self.storage.revision_intrinsic_metadata_add([metadata2_rev])
self.storage.origin_intrinsic_metadata_add([metadata2_origin]) self.storage.origin_intrinsic_metadata_add([metadata2_origin])
# then # then
search = self.storage.origin_intrinsic_metadata_search_fulltext search = self.storage.origin_intrinsic_metadata_search_fulltext
self.assertCountEqual( self.assertCountEqual(
[res['origin_id'] for res in search(['Doe'])], [res['id'] for res in search(['Doe'])],
[self.origin_id_1, self.origin_id_2]) [self.origin_id_1, self.origin_id_2])
self.assertEqual( self.assertEqual(
[res['origin_id'] for res in search(['John', 'Doe'])], [res['id'] for res in search(['John', 'Doe'])],
[self.origin_id_1]) [self.origin_id_1])
self.assertEqual( self.assertEqual(
[res['origin_id'] for res in search(['John'])], [res['id'] for res in search(['John'])],
[self.origin_id_1]) [self.origin_id_1])
self.assertEqual( self.assertEqual(
[res['origin_id'] for res in search(['John', 'Jane'])], [res['id'] for res in search(['John', 'Jane'])],
[]) [])
def test_origin_intrinsic_metadata_search_fulltext_rank(self): def test_origin_intrinsic_metadata_search_fulltext_rank(self):
# given # given
tool_id = self.tools['swh-metadata-detector']['id'] tool_id = self.tools['swh-metadata-detector']['id']
# The following authors have "Random Person" to add some more content # The following authors have "Random Person" to add some more content
# to the JSON data, to work around normalization quirks when there # to the JSON data, to work around normalization quirks when there
# are few words (rank/(1+ln(nb_words)) is very sensitive to nb_words # are few words (rank/(1+ln(nb_words)) is very sensitive to nb_words
# for small values of nb_words). # for small values of nb_words).
metadata1 = { metadata1 = {
'author': [ 'author': [
'Random Person', 'Random Person',
'John Doe', 'John Doe',
'Jane Doe', 'Jane Doe',
] ]
} }
metadata1_rev = { metadata1_rev = {
'id': self.revision_id_1, 'id': self.revision_id_1,
'translated_metadata': metadata1, 'metadata': metadata1,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata1_origin = { metadata1_origin = {
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'metadata': metadata1, 'metadata': metadata1,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'from_revision': self.revision_id_1, 'from_revision': self.revision_id_1,
} }
metadata2 = { metadata2 = {
'author': [ 'author': [
'Random Person', 'Random Person',
'Jane Doe', 'Jane Doe',
] ]
} }
metadata2_rev = { metadata2_rev = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'translated_metadata': metadata2, 'metadata': metadata2,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
} }
metadata2_origin = { metadata2_origin = {
'origin_id': self.origin_id_2, 'id': self.origin_id_2,
'metadata': metadata2, 'metadata': metadata2,
'mappings': [], 'mappings': [],
'indexer_configuration_id': tool_id, 'indexer_configuration_id': tool_id,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
# when # when
self.storage.revision_metadata_add([metadata1_rev]) self.storage.revision_intrinsic_metadata_add([metadata1_rev])
self.storage.origin_intrinsic_metadata_add([metadata1_origin]) self.storage.origin_intrinsic_metadata_add([metadata1_origin])
self.storage.revision_metadata_add([metadata2_rev]) self.storage.revision_intrinsic_metadata_add([metadata2_rev])
self.storage.origin_intrinsic_metadata_add([metadata2_origin]) self.storage.origin_intrinsic_metadata_add([metadata2_origin])
# then # then
search = self.storage.origin_intrinsic_metadata_search_fulltext search = self.storage.origin_intrinsic_metadata_search_fulltext
self.assertEqual( self.assertEqual(
[res['origin_id'] for res in search(['Doe'])], [res['id'] for res in search(['Doe'])],
[self.origin_id_1, self.origin_id_2]) [self.origin_id_1, self.origin_id_2])
self.assertEqual( self.assertEqual(
[res['origin_id'] for res in search(['Doe'], limit=1)], [res['id'] for res in search(['Doe'], limit=1)],
[self.origin_id_1]) [self.origin_id_1])
self.assertEqual( self.assertEqual(
[res['origin_id'] for res in search(['John'])], [res['id'] for res in search(['John'])],
[self.origin_id_1]) [self.origin_id_1])
self.assertEqual( self.assertEqual(
[res['origin_id'] for res in search(['Jane'])], [res['id'] for res in search(['Jane'])],
[self.origin_id_2, self.origin_id_1]) [self.origin_id_2, self.origin_id_1])
self.assertEqual( self.assertEqual(
[res['origin_id'] for res in search(['John', 'Jane'])], [res['id'] for res in search(['John', 'Jane'])],
[self.origin_id_1]) [self.origin_id_1])
def _fill_origin_intrinsic_metadata(self): def _fill_origin_intrinsic_metadata(self):
tool1_id = self.tools['swh-metadata-detector']['id'] tool1_id = self.tools['swh-metadata-detector']['id']
tool2_id = self.tools['swh-metadata-detector2']['id'] tool2_id = self.tools['swh-metadata-detector2']['id']
metadata1 = { metadata1 = {
'@context': 'foo', '@context': 'foo',
'author': 'John Doe', 'author': 'John Doe',
} }
metadata1_rev = { metadata1_rev = {
'id': self.revision_id_1, 'id': self.revision_id_1,
'translated_metadata': metadata1, 'metadata': metadata1,
'mappings': ['npm'], 'mappings': ['npm'],
'indexer_configuration_id': tool1_id, 'indexer_configuration_id': tool1_id,
} }
metadata1_origin = { metadata1_origin = {
'origin_id': self.origin_id_1, 'id': self.origin_id_1,
'metadata': metadata1, 'metadata': metadata1,
'mappings': ['npm'], 'mappings': ['npm'],
'indexer_configuration_id': tool1_id, 'indexer_configuration_id': tool1_id,
'from_revision': self.revision_id_1, 'from_revision': self.revision_id_1,
} }
metadata2 = { metadata2 = {
'@context': 'foo', '@context': 'foo',
'author': 'Jane Doe', 'author': 'Jane Doe',
} }
metadata2_rev = { metadata2_rev = {
'id': self.revision_id_2, 'id': self.revision_id_2,
'translated_metadata': metadata2, 'metadata': metadata2,
'mappings': ['npm', 'gemspec'], 'mappings': ['npm', 'gemspec'],
'indexer_configuration_id': tool2_id, 'indexer_configuration_id': tool2_id,
} }
metadata2_origin = { metadata2_origin = {
'origin_id': self.origin_id_2, 'id': self.origin_id_2,
'metadata': metadata2, 'metadata': metadata2,
'mappings': ['npm', 'gemspec'], 'mappings': ['npm', 'gemspec'],
'indexer_configuration_id': tool2_id, 'indexer_configuration_id': tool2_id,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
} }
metadata3 = { metadata3 = {
'@context': 'foo', '@context': 'foo',
} }
metadata3_rev = { metadata3_rev = {
'id': self.revision_id_3, 'id': self.revision_id_3,
'translated_metadata': metadata3, 'metadata': metadata3,
'mappings': ['npm', 'gemspec'], 'mappings': ['npm', 'gemspec'],
'indexer_configuration_id': tool2_id, 'indexer_configuration_id': tool2_id,
} }
metadata3_origin = { metadata3_origin = {
'origin_id': self.origin_id_3, 'id': self.origin_id_3,
'metadata': metadata3, 'metadata': metadata3,
'mappings': ['pkg-info'], 'mappings': ['pkg-info'],
'indexer_configuration_id': tool2_id, 'indexer_configuration_id': tool2_id,
'from_revision': self.revision_id_3, 'from_revision': self.revision_id_3,
} }
self.storage.revision_metadata_add([metadata1_rev]) self.storage.revision_intrinsic_metadata_add([metadata1_rev])
self.storage.origin_intrinsic_metadata_add([metadata1_origin]) self.storage.origin_intrinsic_metadata_add([metadata1_origin])
self.storage.revision_metadata_add([metadata2_rev]) self.storage.revision_intrinsic_metadata_add([metadata2_rev])
self.storage.origin_intrinsic_metadata_add([metadata2_origin]) self.storage.origin_intrinsic_metadata_add([metadata2_origin])
self.storage.revision_metadata_add([metadata3_rev]) self.storage.revision_intrinsic_metadata_add([metadata3_rev])
self.storage.origin_intrinsic_metadata_add([metadata3_origin]) self.storage.origin_intrinsic_metadata_add([metadata3_origin])
def test_origin_intrinsic_metadata_search_by_producer(self): def test_origin_intrinsic_metadata_search_by_producer(self):
self._fill_origin_intrinsic_metadata() self._fill_origin_intrinsic_metadata()
tool1 = self.tools['swh-metadata-detector'] tool1 = self.tools['swh-metadata-detector']
tool2 = self.tools['swh-metadata-detector2'] tool2 = self.tools['swh-metadata-detector2']
endpoint = self.storage.origin_intrinsic_metadata_search_by_producer endpoint = self.storage.origin_intrinsic_metadata_search_by_producer
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines def test_origin_intrinsic_metadata_search_by_producer(self):
endpoint(tool_ids=[tool2['id']], ids_only=True), endpoint(tool_ids=[tool2['id']], ids_only=True),
[self.origin_id_2, self.origin_id_3]) [self.origin_id_2, self.origin_id_3])
self.assertCountEqual( self.assertCountEqual(
endpoint(tool_ids=[tool1['id'], tool2['id']], ids_only=True), endpoint(tool_ids=[tool1['id'], tool2['id']], ids_only=True),
[self.origin_id_1, self.origin_id_2, self.origin_id_3]) [self.origin_id_1, self.origin_id_2, self.origin_id_3])
# test ids_only=False # test ids_only=False
self.assertEqual(list(endpoint(mappings=['gemspec'])), [{ self.assertEqual(list(endpoint(mappings=['gemspec'])), [{
'origin_id': self.origin_id_2, 'id': self.origin_id_2,
'metadata': { 'metadata': {
'@context': 'foo', '@context': 'foo',
'author': 'Jane Doe', 'author': 'Jane Doe',
}, },
'mappings': ['npm', 'gemspec'], 'mappings': ['npm', 'gemspec'],
'tool': tool2, 'tool': tool2,
'from_revision': self.revision_id_2, 'from_revision': self.revision_id_2,
}]) }])
▲ Show 20 Lines • Show All 356 Lines • Show Last 20 Lines
About · Developer information · Legal