Differential D1080 Diff 3393 swh/indexer/tests/test_origin_metadata.py

Changeset View

Standalone View

swh/indexer/tests/test_origin_metadata.py

# Copyright (C) 2018-2019 The Software Heritage developers		# Copyright (C) 2018-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import pytest		import functools

from unittest import mock		from unittest import mock

from swh.objstorage.objstorage_in_memory import InMemoryObjStorage		from swh.objstorage.objstorage_in_memory import InMemoryObjStorage
from swh.model.hashutil import hash_to_bytes		from swh.model.hashutil import hash_to_bytes
from swh.storage.in_memory import Storage		from swh.storage.in_memory import Storage

from swh.indexer.metadata import OriginMetadataIndexer		from swh.indexer.metadata import OriginMetadataIndexer
from swh.indexer.storage.in_memory import IndexerStorage		from swh.indexer.storage.in_memory import IndexerStorage
Show All 10 Lines	'tools': {
'configuration': {},		'configuration': {},
},		},
'tasks': {		'tasks': {
'revision_metadata': 'revision_metadata',		'revision_metadata': 'revision_metadata',
'origin_intrinsic_metadata': 'origin_intrinsic_metadata',		'origin_intrinsic_metadata': 'origin_intrinsic_metadata',
}		}
}		}

		EXPECTED_METADATA = {
		'@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
		'url':
		'https://github.com/librariesio/yarn-parser#readme',
		'codeRepository':
		'git+git+https://github.com/librariesio/yarn-parser.git',
		'author': [{
		'type': 'Person',
		'name': 'Andrew Nesbitt'
		}],
		'license': 'https://spdx.org/licenses/AGPL-3.0',
		'version': '1.0.0',
		'description':
		'Tiny web service for parsing yarn.lock files',
		'issueTracker':
		'https://github.com/librariesio/yarn-parser/issues',
		'name': 'yarn-parser',
		'keywords': ['yarn', 'parse', 'lock', 'dependencies'],
		}


@pytest.mark.db		def origin_metadata_indexer_test(f):
@mock.patch('swh.indexer.metadata.RevisionMetadataIndexer.parse_config_file')		@mock.patch(
@mock.patch('swh.indexer.origin_head.OriginHeadIndexer.parse_config_file')		'swh.indexer.metadata.RevisionMetadataIndexer.parse_config_file')
		@mock.patch(
		'swh.indexer.origin_head.OriginHeadIndexer.parse_config_file')
@mock.patch('swh.indexer.storage.in_memory.IndexerStorage')		@mock.patch('swh.indexer.storage.in_memory.IndexerStorage')
@mock.patch('swh.storage.in_memory.Storage')		@mock.patch('swh.storage.in_memory.Storage')
def test_full_origin_metadata_indexer(		@functools.wraps(f)
storage_mock, idx_storage_mock, origin_head_parse_config,		def newf(storage_mock, idx_storage_mock, origin_head_parse_config,
revision_metadata_parse_config):		revision_metadata_parse_config):
# Always returns the same instance of the idx storage, because		# Always returns the same instance of the idx storage, because
# this function is called by each of the three indexers.		# this function is called by each of the three indexers.
objstorage = InMemoryObjStorage()		objstorage = InMemoryObjStorage()
storage = Storage()		storage = Storage()
idx_storage = IndexerStorage()		idx_storage = IndexerStorage()

origin_head_parse_config.return_value = ORIGIN_HEAD_CONFIG		origin_head_parse_config.return_value = ORIGIN_HEAD_CONFIG
revision_metadata_parse_config.return_value = REVISION_METADATA_CONFIG		revision_metadata_parse_config.return_value = REVISION_METADATA_CONFIG
storage_mock.return_value = storage		storage_mock.return_value = storage
idx_storage_mock.return_value = idx_storage		idx_storage_mock.return_value = idx_storage

fill_obj_storage(objstorage)		fill_obj_storage(objstorage)
fill_storage(storage)		fill_storage(storage)

# TODO: find a better way to share the ContentMetadataIndexer use		# TODO: find a better way to share the ContentMetadataIndexer use
# the same objstorage instance.		# the same objstorage instance.
import swh.objstorage		import swh.objstorage
old_inmem_objstorage = swh.objstorage._STORAGE_CLASSES['memory']		old_inmem_objstorage = swh.objstorage._STORAGE_CLASSES['memory']
swh.objstorage._STORAGE_CLASSES['memory'] = lambda: objstorage		swh.objstorage._STORAGE_CLASSES['memory'] = lambda: objstorage
try:		try:
		f(storage, idx_storage)
		finally:
		swh.objstorage._STORAGE_CLASSES['memory'] = old_inmem_objstorage

		return newf


		@origin_metadata_indexer_test
		def test_origin_metadata_indexer(storage, idx_storage):
indexer = OriginMetadataIndexer()		indexer = OriginMetadataIndexer()
indexer.storage = storage		indexer.storage = storage
indexer.idx_storage = idx_storage		indexer.idx_storage = idx_storage
indexer.run(["git+https://github.com/librariesio/yarn-parser"])		indexer.run(["git+https://github.com/librariesio/yarn-parser"])
finally:
swh.objstorage._STORAGE_CLASSES['memory'] = old_inmem_objstorage

origin = storage.origin_get({		origin = storage.origin_get({
'type': 'git',		'type': 'git',
'url': 'https://github.com/librariesio/yarn-parser'})		'url': 'https://github.com/librariesio/yarn-parser'})
rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f')		rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f')

metadata = {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'url':
'https://github.com/librariesio/yarn-parser#readme',
'codeRepository':
'git+git+https://github.com/librariesio/yarn-parser.git',
'author': [{
'type': 'Person',
'name': 'Andrew Nesbitt'
}],
'license': 'https://spdx.org/licenses/AGPL-3.0',
'version': '1.0.0',
'description':
'Tiny web service for parsing yarn.lock files',
'issueTracker':
'https://github.com/librariesio/yarn-parser/issues',
'name': 'yarn-parser',
'keywords': ['yarn', 'parse', 'lock', 'dependencies'],
}
rev_metadata = {		rev_metadata = {
'id': rev_id,		'id': rev_id,
'translated_metadata': metadata,		'translated_metadata': EXPECTED_METADATA,
'mappings': ['npm'],		'mappings': ['npm'],
}		}
origin_metadata = {		origin_metadata = {
'origin_id': origin['id'],		'origin_id': origin['id'],
'from_revision': rev_id,		'from_revision': rev_id,
'metadata': metadata,		'metadata': EXPECTED_METADATA,
'mappings': ['npm'],		'mappings': ['npm'],
}		}

results = list(indexer.idx_storage.revision_metadata_get([rev_id]))		results = list(indexer.idx_storage.revision_metadata_get([rev_id]))
for result in results:		for result in results:
del result['tool']		del result['tool']
assert results == [rev_metadata]		assert results == [rev_metadata]

results = list(indexer.idx_storage.origin_intrinsic_metadata_get([		results = list(indexer.idx_storage.origin_intrinsic_metadata_get([
origin['id']]))		origin['id']]))
for result in results:		for result in results:
del result['tool']		del result['tool']
assert results == [origin_metadata]		assert results == [origin_metadata]