Page MenuHomeSoftware Heritage
Differential D952 Diff 3043 swh/indexer/tests/test_origin_metadata.py

Changeset View

Standalone View

View Options

swh/indexer/tests/test_origin_metadata.py

# Copyright (C) 2018 The Software Heritage developers # Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import time from celery.result import AsyncResult
import unittest from unittest import mock
import unittest.mock
from celery import task
from swh.model.hashutil import hash_to_bytes from swh.model.hashutil import hash_to_bytes
from swh.storage.in_memory import Storage from swh.storage.in_memory import Storage
from swh.indexer.metadata import (
OriginMetadataIndexer, RevisionMetadataIndexer
)
from swh.indexer.storage.in_memory import IndexerStorage from swh.indexer.storage.in_memory import IndexerStorage
from swh.objstorage.objstorage_in_memory import InMemoryObjStorage from swh.objstorage.objstorage_in_memory import InMemoryObjStorage
from swh.scheduler.tests.scheduler_testing import SchedulerTestFixture from swh.scheduler.celery_backend.runner import run_ready_tasks
from .test_utils import (
BASE_TEST_CONFIG, fill_storage, fill_obj_storage
)
from .test_origin_head import OriginHeadTestIndexer
from .test_metadata import ContentMetadataTestIndexer
class RevisionMetadataTestIndexer(RevisionMetadataIndexer):
"""Specific indexer whose configuration is enough to satisfy the
indexing tests.
"""
ContentMetadataIndexer = ContentMetadataTestIndexer
def parse_config_file(self, *args, **kwargs):
return {
**BASE_TEST_CONFIG,
'tools': {
'name': 'swh-metadata-detector',
'version': '0.0.2',
'configuration': {
'type': 'local',
'context': 'NpmMapping'
}
}
}
@task
def revision_metadata_test_task(*args, **kwargs):
indexer = RevisionMetadataTestIndexer()
indexer.run(*args, **kwargs)
return indexer.results
class OriginMetadataTestIndexer(OriginMetadataIndexer): from .test_utils import fill_storage, fill_obj_storage
def parse_config_file(self, *args, **kwargs): from .test_origin_head import OriginHeadTestIndexer
return { from swh.indexer.tests.tasks import (
**BASE_TEST_CONFIG, RevisionMetadataTestIndexer, OriginMetadataTestIndexer)
'tools': []
}
@task
def origin_intrinsic_metadata_test_task(*args, **kwargs):
indexer = OriginMetadataTestIndexer()
indexer.run(*args, **kwargs)
return indexer.results
class OriginHeadTestIndexer(OriginHeadTestIndexer): class OriginHeadTestIndexer(OriginHeadTestIndexer):
def prepare(self): def prepare(self):
super().prepare() super().prepare()
self.config['tasks'] = { self.config['tasks'] = {
'revision_metadata': 'revision_metadata_test_task', 'revision_metadata': 'revision_metadata',
'origin_intrinsic_metadata': 'origin_intrinsic_metadata_test_task', 'origin_intrinsic_metadata': 'origin_intrinsic_metadata',
} }
class TestOriginMetadata(SchedulerTestFixture, unittest.TestCase): @mock.patch('swh.indexer.storage.in_memory.IndexerStorage')
def setUp(self): @mock.patch('swh.storage.in_memory.Storage')
super().setUp() def test_pipeline(storage_mock, idx_storage_mock,
self.maxDiff = None swh_app, celery_session_worker, indexer_scheduler):
self.add_scheduler_task_type( scheduler = indexer_scheduler
'revision_metadata_test_task',
'swh.indexer.tests.test_origin_metadata.'
'revision_metadata_test_task')
self.add_scheduler_task_type(
'origin_intrinsic_metadata_test_task',
'swh.indexer.tests.test_origin_metadata.'
'origin_intrinsic_metadata_test_task')
RevisionMetadataTestIndexer.scheduler = self.scheduler
def tearDown(self):
del RevisionMetadataTestIndexer.scheduler
super().tearDown()
@unittest.mock.patch('swh.indexer.storage.in_memory.IndexerStorage')
@unittest.mock.patch('swh.storage.in_memory.Storage')
def test_pipeline(self, storage_mock, idx_storage_mock):
# Always returns the same instance of the idx storage, because # Always returns the same instance of the idx storage, because
# this function is called by each of the three indexers. # this function is called by each of the three indexers.
objstorage = InMemoryObjStorage() objstorage = InMemoryObjStorage()
storage = Storage() storage = Storage()
idx_storage = IndexerStorage() idx_storage = IndexerStorage()
storage_mock.return_value = storage storage_mock.return_value = storage
idx_storage_mock.return_value = idx_storage idx_storage_mock.return_value = idx_storage
fill_obj_storage(objstorage) fill_obj_storage(objstorage)
fill_storage(storage) fill_storage(storage)
# TODO: find a better way to share the ContentMetadataIndexer use # TODO: find a better way to share the ContentMetadataIndexer use
# the same objstorage instance. # the same objstorage instance.
import swh.objstorage import swh.objstorage
old_inmem_objstorage = swh.objstorage._STORAGE_CLASSES['memory'] old_inmem_objstorage = swh.objstorage._STORAGE_CLASSES['memory']
swh.objstorage._STORAGE_CLASSES['memory'] = lambda: objstorage swh.objstorage._STORAGE_CLASSES['memory'] = lambda: objstorage
try: try:
RevisionMetadataTestIndexer.scheduler = scheduler
OriginMetadataTestIndexer.scheduler = scheduler
indexer = OriginHeadTestIndexer() indexer = OriginHeadTestIndexer()
indexer.scheduler = self.scheduler indexer.scheduler = scheduler
indexer.run(["git+https://github.com/librariesio/yarn-parser"]) indexer.run(["git+https://github.com/librariesio/yarn-parser"])
self.run_ready_tasks() # Run the first task tasks = run_ready_tasks(scheduler, swh_app) # Run the first task
# Give it time to complete and schedule the 2nd one # Wait for the task to complete and schedule the 2nd one
time.sleep(0.1) assert len(tasks) == 1
self.run_ready_tasks() # Run the second task promise = AsyncResult(id=tasks[0]['backend_id'])
promise.wait()
tasks = run_ready_tasks(scheduler, swh_app) # Run the second task
assert len(tasks) == 1
promise = AsyncResult(id=tasks[0]['backend_id'])
promise.wait()
finally: finally:
swh.objstorage._STORAGE_CLASSES['memory'] = old_inmem_objstorage swh.objstorage._STORAGE_CLASSES['memory'] = old_inmem_objstorage
del RevisionMetadataTestIndexer.scheduler
del OriginMetadataTestIndexer.scheduler
origin = storage.origin_get({ origin = storage.origin_get({
'type': 'git', 'type': 'git',
'url': 'https://github.com/librariesio/yarn-parser'}) 'url': 'https://github.com/librariesio/yarn-parser'})
rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f')
metadata = { metadata = {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'url': 'url':
'https://github.com/librariesio/yarn-parser#readme', 'https://github.com/librariesio/yarn-parser#readme',
'codeRepository': 'codeRepository':
'git+git+https://github.com/librariesio/yarn-parser.git', 'git+git+https://github.com/librariesio/yarn-parser.git',
'author': [{ 'author': [{
'type': 'Person', 'type': 'Person',
'name': 'Andrew Nesbitt' 'name': 'Andrew Nesbitt'
}], }],
'license': 'https://spdx.org/licenses/AGPL-3.0', 'license': 'https://spdx.org/licenses/AGPL-3.0',
'version': '1.0.0', 'version': '1.0.0',
'description': 'description':
'Tiny web service for parsing yarn.lock files', 'Tiny web service for parsing yarn.lock files',
'issueTracker': 'issueTracker':
'https://github.com/librariesio/yarn-parser/issues', 'https://github.com/librariesio/yarn-parser/issues',
'name': 'yarn-parser', 'name': 'yarn-parser',
'keywords': ['yarn', 'parse', 'lock', 'dependencies'], 'keywords': ['yarn', 'parse', 'lock', 'dependencies'],
} }
rev_metadata = { rev_metadata = {
'id': rev_id, 'id': rev_id,
'translated_metadata': metadata, 'translated_metadata': metadata,
} }
origin_metadata = { origin_metadata = {
'origin_id': origin['id'], 'origin_id': origin['id'],
'from_revision': rev_id, 'from_revision': rev_id,
'metadata': metadata, 'metadata': metadata,
} }
results = list(indexer.idx_storage.revision_metadata_get([rev_id])) results = list(indexer.idx_storage.revision_metadata_get([rev_id]))
for result in results: for result in results:
del result['tool'] del result['tool']
self.assertEqual(results, [rev_metadata]) assert results == [rev_metadata]
results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ results = list(indexer.idx_storage.origin_intrinsic_metadata_get([
origin['id']])) origin['id']]))
for result in results: for result in results:
del result['tool'] del result['tool']
self.assertEqual(results, [origin_metadata]) assert results == [origin_metadata]
About · Developer information · Legal