diff --git a/swh/lister/core/tests/conftest.py b/swh/lister/core/tests/conftest.py index 3961386..b7093d8 100644 --- a/swh/lister/core/tests/conftest.py +++ b/swh/lister/core/tests/conftest.py @@ -1,39 +1,47 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.scheduler.tests.conftest import * # noqa import logging import pytest from sqlalchemy import create_engine from swh.lister import get_lister, SUPPORTED_LISTERS from swh.lister.core.models import initialize logger = logging.getLogger(__name__) @pytest.fixture def swh_listers(request, postgresql_proc, postgresql, swh_scheduler): db_url = 'postgresql://{user}@{host}:{port}/{dbname}'.format( host=postgresql_proc.host, port=postgresql_proc.port, user='postgres', dbname='tests') logger.debug('lister db_url: %s', db_url) listers = {} # Prepare schema for all listers for lister_name in SUPPORTED_LISTERS: lister = get_lister(lister_name, db_url=db_url) lister.scheduler = swh_scheduler # inject scheduler fixture listers[lister_name] = lister initialize(create_engine(db_url), drop_tables=True) + # Add the load-archive-files expected by some listers (gnu, cran, ...) + swh_scheduler.create_task_type({ + 'type': 'load-archive-files', + 'description': 'Load archive files.', + 'backend_name': 'swh.loader.package.tasks.LoadArchive', + 'default_interval': '1 day', + }) + return listers diff --git a/swh/lister/cran/tests/conftest.py b/swh/lister/cran/tests/conftest.py index 849dab2..a4b2b26 100644 --- a/swh/lister/cran/tests/conftest.py +++ b/swh/lister/cran/tests/conftest.py @@ -1,23 +1,6 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import pytest - from swh.lister.core.tests.conftest import * # noqa - - -@pytest.fixture -def lister_cran(swh_listers): - lister = swh_listers['cran'] - - # Add the load-deb-package in the scheduler backend - lister.scheduler.create_task_type({ - 'type': 'load-archive-files', - 'description': 'Load archive files.', - 'backend_name': 'swh.loader.package.tasks.LoadArchive', - 'default_interval': '1 day', - }) - - return lister diff --git a/swh/lister/cran/tests/test_lister.py b/swh/lister/cran/tests/test_lister.py index 3e7a41a..b02da46 100644 --- a/swh/lister/cran/tests/test_lister.py +++ b/swh/lister/cran/tests/test_lister.py @@ -1,63 +1,65 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import json import pytest from os import path from unittest.mock import patch from swh.lister.cran.lister import compute_package_url def test_cran_compute_package_url(): url = compute_package_url({'Package': 'something', 'Version': '0.0.1'}) assert url == 'https://cran.r-project.org/src/contrib/%s_%s.tar.gz' % ( 'something', '0.0.1', ) def test_cran_compute_package_url_failure(): for incomplete_repo in [{'Version': '0.0.1'}, {'Package': 'package'}, {}]: with pytest.raises(KeyError): compute_package_url(incomplete_repo) @patch('swh.lister.cran.lister.read_cran_data') -def test_cran_lister_cran(mock_cran, datadir, lister_cran): +def test_cran_lister_cran(mock_cran, datadir, swh_listers): + lister = swh_listers['cran'] + with open(path.join(datadir, 'list-r-packages.json')) as f: data = json.loads(f.read()) mock_cran.return_value = data assert len(data) == 6 - lister_cran.run() + lister.run() - r = lister_cran.scheduler.search_tasks(task_type='load-archive-files') + r = lister.scheduler.search_tasks(task_type='load-archive-files') assert len(r) == 6 for row in r: assert row['type'] == 'load-archive-files' # arguments check args = row['arguments']['args'] assert len(args) == 3 # ['SeleMix', # 'https://cran.r-project.org/src/contrib/SeleMix_1.0.1.tar.gz', # '1.0.1'] package = args[0] url = args[1] version = args[2] assert url == compute_package_url( {'Package': package, 'Version': version}) # kwargs kwargs = row['arguments']['kwargs'] assert kwargs == {} assert row['policy'] == 'oneshot' diff --git a/swh/lister/gnu/tests/conftest.py b/swh/lister/gnu/tests/conftest.py index ee463a6..a4b2b26 100644 --- a/swh/lister/gnu/tests/conftest.py +++ b/swh/lister/gnu/tests/conftest.py @@ -1,23 +1,6 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import pytest - from swh.lister.core.tests.conftest import * # noqa - - -@pytest.fixture -def lister_gnu(swh_listers): - lister = swh_listers['gnu'] - - # Add the load-deb-package in the scheduler backend - lister.scheduler.create_task_type({ - 'type': 'load-archive-files', - 'description': 'Load archive files.', - 'backend_name': 'swh.loader.package.tasks.LoadArchive', - 'default_interval': '1 day', - }) - - return lister diff --git a/swh/lister/gnu/tests/test_lister.py b/swh/lister/gnu/tests/test_lister.py index 781bad3..a1c9c09 100644 --- a/swh/lister/gnu/tests/test_lister.py +++ b/swh/lister/gnu/tests/test_lister.py @@ -1,50 +1,52 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging logger = logging.getLogger(__name__) -def test_gnu_lister(lister_gnu, requests_mock_datadir): - lister_gnu.run() +def test_gnu_lister(swh_listers, requests_mock_datadir): + lister = swh_listers['gnu'] - r = lister_gnu.scheduler.search_tasks(task_type='load-archive-files') + lister.run() + + r = lister.scheduler.search_tasks(task_type='load-archive-files') assert len(r) == 383 for row in r: assert row['type'] == 'load-archive-files' # arguments check args = row['arguments']['args'] assert len(args) == 0 # kwargs kwargs = row['arguments']['kwargs'] assert set(kwargs.keys()) == {'url', 'artifacts'} url = kwargs['url'] assert url.startswith('https://ftp.gnu.org') url_suffix = url.split('https://ftp.gnu.org')[1] assert 'gnu' in url_suffix or 'old-gnu' in url_suffix artifacts = kwargs['artifacts'] # check the artifact's structure artifact = artifacts[0] assert set(artifact.keys()) == { 'url', 'length', 'time', 'filename', 'version' } for artifact in artifacts: logger.debug(artifact) # 'time' is an isoformat string now for key in ['url', 'time', 'filename', 'version']: assert isinstance(artifact[key], str) assert isinstance(artifact['length'], int) assert row['policy'] == 'oneshot' assert row['priority'] is None assert row['retries_left'] == 3