diff --git a/swh/lister/cran/lister.py b/swh/lister/cran/lister.py --- a/swh/lister/cran/lister.py +++ b/swh/lister/cran/lister.py @@ -8,7 +8,7 @@ import pkg_resources import subprocess -from typing import List, Mapping +from typing import List, Mapping, Tuple from swh.lister.cran.models import CRANModel @@ -19,28 +19,39 @@ logger = logging.getLogger(__name__) +CRAN_MIRROR = 'https://cran.r-project.org' + + class CRANLister(SimpleLister): MODEL = CRANModel LISTER_NAME = 'cran' instance = 'cran' - def task_dict(self, origin_type, origin_url, **kwargs): + def task_dict(self, origin_type, origin_url, version=None, html_url=None, + policy=None, **kwargs): """Return task format dict. This creates tasks with args and kwargs set, for example:: args: [] kwargs: { - 'url': 'https://cran.r-project.org/...', - 'version': '0.0.1' + 'url': 'https://cran.r-project.org/Packages/...', + 'artifacts': [{ + 'url': 'https://cran.r-project.org/...', + 'version': '0.0.1', + }] } """ - policy = kwargs.get('policy', 'oneshot') - version = kwargs.get('version') + if not policy: + policy = 'oneshot' + artifact_url = html_url assert origin_type == 'tar' return create_task_dict( 'load-cran', policy, - url=origin_url, version=version, retries_left=3 + url=origin_url, artifacts=[{ + 'url': artifact_url, + 'version': version + }], retries_left=3 ) def safely_issue_request(self, identifier): @@ -95,15 +106,15 @@ """ logger.debug('repo: %s', repo) - project_url = compute_package_url(repo) + origin_url, artifact_url = compute_origin_urls(repo) package = repo['Package'] return { 'uid': package, 'name': package, 'full_name': repo['Title'], 'version': repo['Version'], - 'html_url': project_url, - 'origin_url': project_url, + 'html_url': artifact_url, + 'origin_url': origin_url, 'origin_type': 'tar', } @@ -119,15 +130,18 @@ return json.loads(response.stdout.decode('utf-8')) -def compute_package_url(repo: Mapping[str, str]) -> str: +def compute_origin_urls(repo: Mapping[str, str]) -> Tuple[str, str]: """Compute the package url from the repo dict. Args: repo: dict with key 'Package', 'Version' Returns: - the package url + the tuple project url, artifact url """ - return 'https://cran.r-project.org/src/contrib' \ - '/{Package}_{Version}.tar.gz'.format(**repo) + package = repo['Package'] + version = repo['Version'] + origin_url = f'{CRAN_MIRROR}/package={package}' + artifact_url = f'{CRAN_MIRROR}/src/contrib/{package}_{version}.tar.gz' + return origin_url, artifact_url diff --git a/swh/lister/cran/tests/test_lister.py b/swh/lister/cran/tests/test_lister.py --- a/swh/lister/cran/tests/test_lister.py +++ b/swh/lister/cran/tests/test_lister.py @@ -9,22 +9,25 @@ from os import path from unittest.mock import patch -from swh.lister.cran.lister import compute_package_url +from swh.lister.cran.lister import compute_origin_urls, CRAN_MIRROR -def test_cran_compute_package_url(): - url = compute_package_url({'Package': 'something', 'Version': '0.0.1'}) +def test_cran_compute_origin_urls(): + pack = 'something' + vers = '0.0.1' + origin_url, artifact_url = compute_origin_urls({ + 'Package': pack, + 'Version': vers, + }) - assert url == 'https://cran.r-project.org/src/contrib/%s_%s.tar.gz' % ( - 'something', - '0.0.1', - ) + assert origin_url == f'{CRAN_MIRROR}/package={pack}' + assert artifact_url == f'{CRAN_MIRROR}/src/contrib/{pack}_{vers}.tar.gz' -def test_cran_compute_package_url_failure(): +def test_cran_compute_origin_urls_failure(): for incomplete_repo in [{'Version': '0.0.1'}, {'Package': 'package'}, {}]: with pytest.raises(KeyError): - compute_package_url(incomplete_repo) + compute_origin_urls(incomplete_repo) @patch('swh.lister.cran.lister.read_cran_data') @@ -51,7 +54,12 @@ # kwargs kwargs = row['arguments']['kwargs'] assert len(kwargs) == 2 - assert set(kwargs.keys()) == {'url', 'version'} + assert set(kwargs.keys()) == {'url', 'artifacts'} + + artifacts = kwargs['artifacts'] + assert len(artifacts) == 1 + + assert set(artifacts[0].keys()) == {'url', 'version'} assert row['policy'] == 'oneshot' assert row['retries_left'] == 3