Differential D742 Diff 2339 swh/indexer/tests/test_origin_metadata.py

Changeset View

Standalone View

swh/indexer/tests/test_origin_metadata.py

# Copyright (C) 2018 The Software Heritage developers		# Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import time		import time
import logging
import unittest		import unittest
from celery import task

from swh.indexer.metadata import OriginMetadataIndexer, \
RevisionMetadataIndexer, ContentMetadataIndexer
from swh.indexer.tests.test_utils import MockObjStorage, MockStorage
from swh.indexer.tests.test_utils import MockIndexerStorage
from swh.indexer.tests.test_origin_head import OriginHeadTestIndexer

from swh.scheduler.tests.scheduler_testing import SchedulerTestFixture

		from celery import task
from swh.model.hashutil import hash_to_bytes		from swh.model.hashutil import hash_to_bytes

from .test_utils import BASE_TEST_CONFIG		from swh.indexer.metadata import (
		OriginMetadataIndexer, RevisionMetadataIndexer
		)

class OriginMetadataTestIndexer(OriginMetadataIndexer):		from swh.scheduler.tests.scheduler_testing import SchedulerTestFixture
def parse_config_file(self, args, *kwargs):		from .test_utils import (
return {		MockObjStorage, MockStorage, MockIndexerStorage,
**BASE_TEST_CONFIG,		BASE_TEST_CONFIG
'tools': [],		)
}		from .test_origin_head import OriginHeadTestIndexer
		from .test_metadata import ContentMetadataTestIndexer
def prepare(self):
super().prepare()
self.storage = MockStorage()
self.idx_storage = MockIndexerStorage()
self.objstorage = MockObjStorage()


class ContentMetadataTestIndexer(ContentMetadataIndexer):
"""Specific Metadata whose configuration is enough to satisfy the
indexing tests.
"""
def prepare(self):
self.idx_storage = MockIndexerStorage()
self.log = logging.getLogger('swh.indexer')
self.objstorage = MockObjStorage()
self.tools = self.register_tools(self.config['tools'])
self.tool = self.tools[0]
self.results = []


class RevisionMetadataTestIndexer(RevisionMetadataIndexer):		class RevisionMetadataTestIndexer(RevisionMetadataIndexer):
"""Specific indexer whose configuration is enough to satisfy the		"""Specific indexer whose configuration is enough to satisfy the
indexing tests.		indexing tests.
"""		"""

ContentMetadataIndexer = ContentMetadataTestIndexer		ContentMetadataIndexer = ContentMetadataTestIndexer

def prepare(self):		def parse_config_file(self, args, *kwargs):
self.config = {		return {
'storage': {},		**BASE_TEST_CONFIG,
'objstorage': {},
'indexer_storage': {},
'tools': {		'tools': {
'name': 'swh-metadata-detector',		'name': 'swh-metadata-detector',
'version': '0.0.2',		'version': '0.0.2',
'configuration': {		'configuration': {
'type': 'local',		'type': 'local',
'context': 'NpmMapping'		'context': 'NpmMapping'
}		}
}		}
}		}
self.storage = MockStorage()
		def prepare(self):
		super().prepare()
self.idx_storage = MockIndexerStorage()		self.idx_storage = MockIndexerStorage()
self.log = logging.getLogger('swh.indexer')		self.storage = MockStorage()
self.objstorage = MockObjStorage()		self.objstorage = MockObjStorage()
self.tools = self.register_tools(self.config['tools'])
self.tool = self.tools[0]

		ardumontAuthorUnsubmitted Done Inline Actions but no! that was a tryout. ardumont: but no! that was a tryout.
		vlorentzUnsubmitted Not Done Inline Actions ? vlorentz: ?
		ardumontAuthorUnsubmitted Done Inline Actions revision_metadata_indexer = RevisionMetadataTestIndexer() which has come back and needs to go. I'll update the diff again. ardumont: > revision_metadata_indexer = RevisionMetadataTestIndexer() which has come back and needs to…
		vlorentzUnsubmitted Not Done Inline Actions sorry :/ vlorentz: sorry :/
		ardumontAuthorUnsubmitted Done Inline Actions life happens, don't worry ;) ardumont: life happens, don't worry ;)

@task		@task
def revision_metadata_test_task(args, *kwargs):		def revision_metadata_test_task(args, *kwargs):
indexer = RevisionMetadataTestIndexer()		indexer = RevisionMetadataTestIndexer()
indexer.run(args, *kwargs)		indexer.run(args, *kwargs)
return indexer.results		return indexer.results


		class OriginMetadataTestIndexer(OriginMetadataIndexer):
		def parse_config_file(self, args, *kwargs):
		return {
		**BASE_TEST_CONFIG,
		'tools': []
		}

		def prepare(self):
		super().prepare()
		self.storage = MockStorage()
		self.objstorage = MockObjStorage()
		self.idx_storage = MockIndexerStorage()


@task		@task
def origin_intrinsic_metadata_test_task(args, *kwargs):		def origin_intrinsic_metadata_test_task(args, *kwargs):
indexer = OriginMetadataTestIndexer()		indexer = OriginMetadataTestIndexer()
indexer.run(args, *kwargs)		indexer.run(args, *kwargs)
return indexer.results		return indexer.results


class OriginHeadTestIndexer(OriginHeadTestIndexer):		class OriginHeadTestIndexer(OriginHeadTestIndexer):
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines	def test_pipeline(self):
'codemeta:issueTracker':		'codemeta:issueTracker':
'https://github.com/librariesio/yarn-parser/issues',		'https://github.com/librariesio/yarn-parser/issues',
'name': 'yarn-parser',		'name': 'yarn-parser',
'keywords': ['yarn', 'parse', 'lock', 'dependencies'],		'keywords': ['yarn', 'parse', 'lock', 'dependencies'],
}		}
rev_metadata = {		rev_metadata = {
'id': hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'),		'id': hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'),
'translated_metadata': metadata,		'translated_metadata': metadata,
'indexer_configuration_id': 7,
}		}
origin_metadata = {		origin_metadata = {
'origin_id': 54974445,		'origin_id': 54974445,
'from_revision': hash_to_bytes(		'from_revision': hash_to_bytes(
'8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'),		'8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'),
'metadata': metadata,		'metadata': metadata,
'indexer_configuration_id': 7,
}		}
expected_results = [		expected_results = [
		('revision_metadata', True, [rev_metadata]),
('origin_intrinsic_metadata', True, [origin_metadata]),		('origin_intrinsic_metadata', True, [origin_metadata]),
('revision_metadata', True, [rev_metadata])]		]

results = list(indexer.idx_storage.added_data)		results = list(indexer.idx_storage.added_data)
		for result in results:
		metadata = result[2]
		for item in metadata:
		# cannot check those (generated ids)
		del item['indexer_configuration_id']

self.assertCountEqual(expected_results, results)		self.assertCountEqual(expected_results, results)
		vlorentzUnsubmitted Not Done Inline Actions ^^ vlorentz: ^^