Differential D1106 Diff 3525 swh/indexer/tests/test_origin_metadata.py

Changeset View

Standalone View

swh/indexer/tests/test_origin_metadata.py

	# Copyright (C) 2018-2019 The Software Heritage developers			# Copyright (C) 2018-2019 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution			# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

	import pytest			import pytest

	from unittest import mock			from unittest.mock import patch

	from swh.objstorage.objstorage_in_memory import InMemoryObjStorage
	from swh.model.hashutil import hash_to_bytes			from swh.model.hashutil import hash_to_bytes
	from swh.storage.in_memory import Storage

	from swh.indexer.metadata import OriginMetadataIndexer			from swh.indexer.metadata import OriginMetadataIndexer
	from swh.indexer.storage.in_memory import IndexerStorage

	from .utils import fill_storage, fill_obj_storage, BASE_TEST_CONFIG			from .utils import BASE_TEST_CONFIG, YARN_PARSER_METADATA
	from .test_metadata import REVISION_METADATA_CONFIG			from .test_metadata import REVISION_METADATA_CONFIG


	ORIGIN_HEAD_CONFIG = {			ORIGIN_HEAD_CONFIG = {
	**BASE_TEST_CONFIG,			**BASE_TEST_CONFIG,
	'tools': {			'tools': {
	'name': 'origin-metadata',			'name': 'origin-metadata',
	'version': '0.0.1',			'version': '0.0.1',
	'configuration': {},			'configuration': {},
	},			},
	'tasks': {			'tasks': {
	'revision_metadata': 'revision_metadata',			'revision_metadata': 'revision_metadata',
	'origin_intrinsic_metadata': 'origin_intrinsic_metadata',			'origin_intrinsic_metadata': 'origin_intrinsic_metadata',
	}			}
	}			}


	@pytest.mark.db			@pytest.fixture
	ardumontUnsubmitted Not Done Inline Actions this is voluntary because during the debian build for now, test depending on this won't work. ardumont: this is voluntary because during the debian build for now, test depending on this won't work.
	@mock.patch('swh.indexer.metadata.RevisionMetadataIndexer.parse_config_file')			def origin_metadata_indexer():
	@mock.patch('swh.indexer.origin_head.OriginHeadIndexer.parse_config_file')			prefix = 'swh.indexer.'
	@mock.patch('swh.indexer.storage.in_memory.IndexerStorage')			suffix = '.parse_config_file'
	@mock.patch('swh.storage.in_memory.Storage')			with patch(prefix + 'metadata.OriginMetadataIndexer' + suffix) as omi, \
	def test_full_origin_metadata_indexer(			patch(prefix + 'origin_head.OriginHeadIndexer' + suffix) as ohi, \
	storage_mock, idx_storage_mock, origin_head_parse_config,			patch(prefix + 'metadata.RevisionMetadataIndexer' + suffix) as rmi:
	revision_metadata_parse_config):			omi.return_value = BASE_TEST_CONFIG
	# Always returns the same instance of the idx storage, because			ohi.return_value = ORIGIN_HEAD_CONFIG
	# this function is called by each of the three indexers.			rmi.return_value = REVISION_METADATA_CONFIG
	objstorage = InMemoryObjStorage()			yield OriginMetadataIndexer()
	storage = Storage()
	ardumontUnsubmitted Not Done Inline Actions nevermind, it's indeed an in-memory storage, i missed that initially ;) ardumont: nevermind, it's indeed an in-memory storage, i missed that initially ;)
	idx_storage = IndexerStorage()
				def test_origin_metadata_indexer(
	origin_head_parse_config.return_value = ORIGIN_HEAD_CONFIG			idx_storage, storage, obj_storage, origin_metadata_indexer):
	revision_metadata_parse_config.return_value = REVISION_METADATA_CONFIG
	storage_mock.return_value = storage
	idx_storage_mock.return_value = idx_storage

	fill_obj_storage(objstorage)
	fill_storage(storage)

	# TODO: find a better way to share the ContentMetadataIndexer use
	# the same objstorage instance.
	import swh.objstorage
	old_inmem_objstorage = swh.objstorage._STORAGE_CLASSES['memory']
	swh.objstorage._STORAGE_CLASSES['memory'] = lambda: objstorage
	try:
	indexer = OriginMetadataIndexer()			indexer = OriginMetadataIndexer()
	indexer.storage = storage
	indexer.idx_storage = idx_storage
	indexer.run(["git+https://github.com/librariesio/yarn-parser"])			indexer.run(["git+https://github.com/librariesio/yarn-parser"])
	finally:
	swh.objstorage._STORAGE_CLASSES['memory'] = old_inmem_objstorage

	origin = storage.origin_get({			origin = storage.origin_get({
	'type': 'git',			'type': 'git',
	'url': 'https://github.com/librariesio/yarn-parser'})			'url': 'https://github.com/librariesio/yarn-parser'})
	rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f')			rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f')

	metadata = {
	'@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
	'url':
	'https://github.com/librariesio/yarn-parser#readme',
	'codeRepository':
	'git+git+https://github.com/librariesio/yarn-parser.git',
	'author': [{
	'type': 'Person',
	'name': 'Andrew Nesbitt'
	}],
	'license': 'https://spdx.org/licenses/AGPL-3.0',
	'version': '1.0.0',
	'description':
	'Tiny web service for parsing yarn.lock files',
	'issueTracker':
	'https://github.com/librariesio/yarn-parser/issues',
	'name': 'yarn-parser',
	'keywords': ['yarn', 'parse', 'lock', 'dependencies'],
	}
	rev_metadata = {			rev_metadata = {
	'id': rev_id,			'id': rev_id,
	'translated_metadata': metadata,			'translated_metadata': YARN_PARSER_METADATA,
	'mappings': ['npm'],			'mappings': ['npm'],
	}			}
	origin_metadata = {			origin_metadata = {
	'origin_id': origin['id'],			'origin_id': origin['id'],
	'from_revision': rev_id,			'from_revision': rev_id,
	'metadata': metadata,			'metadata': YARN_PARSER_METADATA,
	'mappings': ['npm'],			'mappings': ['npm'],
	}			}

	results = list(indexer.idx_storage.revision_metadata_get([rev_id]))			results = list(indexer.idx_storage.revision_metadata_get([rev_id]))
	for result in results:			for result in results:
	del result['tool']			del result['tool']
	assert results == [rev_metadata]			assert results == [rev_metadata]

	results = list(indexer.idx_storage.origin_intrinsic_metadata_get([			results = list(indexer.idx_storage.origin_intrinsic_metadata_get([
	origin['id']]))			origin['id']]))
	for result in results:			for result in results:
	del result['tool']			del result['tool']
	assert results == [origin_metadata]			assert results == [origin_metadata]