Page MenuHomeSoftware Heritage
Files F9339673

D2524.id9040.diff
No OneTemporary
Actions

D2524.id9040.diff
View Options

diff --git a/swh/lister/cran/lister.py b/swh/lister/cran/lister.py
--- a/swh/lister/cran/lister.py
+++ b/swh/lister/cran/lister.py
@@ -8,7 +8,7 @@
import pkg_resources
import subprocess
-from typing import List, Mapping
+from typing import List, Mapping, Tuple
from swh.lister.cran.models import CRANModel
@@ -19,28 +19,39 @@
logger = logging.getLogger(__name__)
+CRAN_MIRROR = 'https://cran.r-project.org'
+
+
class CRANLister(SimpleLister):
MODEL = CRANModel
LISTER_NAME = 'cran'
instance = 'cran'
- def task_dict(self, origin_type, origin_url, **kwargs):
+ def task_dict(self, origin_type, origin_url, version=None, html_url=None,
+ policy=None, **kwargs):
"""Return task format dict. This creates tasks with args and kwargs
set, for example::
args: []
kwargs: {
- 'url': 'https://cran.r-project.org/...',
- 'version': '0.0.1'
+ 'url': 'https://cran.r-project.org/Packages/<package>...',
+ 'artifacts': [{
+ 'url': 'https://cran.r-project.org/...',
+ 'version': '0.0.1',
+ }]
}
"""
- policy = kwargs.get('policy', 'oneshot')
- version = kwargs.get('version')
+ if not policy:
+ policy = 'oneshot'
+ artifact_url = html_url
assert origin_type == 'tar'
return create_task_dict(
'load-cran', policy,
- url=origin_url, version=version, retries_left=3
+ url=origin_url, artifacts=[{
+ 'url': artifact_url,
+ 'version': version
+ }], retries_left=3
)
def safely_issue_request(self, identifier):
@@ -95,7 +106,7 @@
"""
logger.debug('repo: %s', repo)
- project_url = compute_package_url(repo)
+ origin_url, artifact_url = compute_origin_urls(repo)
package = repo['Package']
version = repo['Version']
return {
@@ -103,8 +114,8 @@
'name': package,
'full_name': repo['Title'],
'version': version,
- 'html_url': project_url,
- 'origin_url': project_url,
+ 'html_url': artifact_url,
+ 'origin_url': origin_url,
'origin_type': 'tar',
}
@@ -120,15 +131,18 @@
return json.loads(response.stdout.decode('utf-8'))
-def compute_package_url(repo: Mapping[str, str]) -> str:
+def compute_origin_urls(repo: Mapping[str, str]) -> Tuple[str, str]:
"""Compute the package url from the repo dict.
Args:
repo: dict with key 'Package', 'Version'
Returns:
- the package url
+ the tuple project url, artifact url
"""
- return 'https://cran.r-project.org/src/contrib' \
- '/{Package}_{Version}.tar.gz'.format(**repo)
+ package = repo['Package']
+ version = repo['Version']
+ origin_url = f'{CRAN_MIRROR}/package={package}'
+ artifact_url = f'{CRAN_MIRROR}/src/contrib/{package}_{version}.tar.gz'
+ return origin_url, artifact_url
diff --git a/swh/lister/cran/tests/test_lister.py b/swh/lister/cran/tests/test_lister.py
--- a/swh/lister/cran/tests/test_lister.py
+++ b/swh/lister/cran/tests/test_lister.py
@@ -9,22 +9,25 @@
from os import path
from unittest.mock import patch
-from swh.lister.cran.lister import compute_package_url
+from swh.lister.cran.lister import compute_origin_urls, CRAN_MIRROR
-def test_cran_compute_package_url():
- url = compute_package_url({'Package': 'something', 'Version': '0.0.1'})
+def test_cran_compute_origin_urls():
+ pack = 'something'
+ vers = '0.0.1'
+ origin_url, artifact_url = compute_origin_urls({
+ 'Package': pack,
+ 'Version': vers,
+ })
- assert url == 'https://cran.r-project.org/src/contrib/%s_%s.tar.gz' % (
- 'something',
- '0.0.1',
- )
+ assert origin_url == f'{CRAN_MIRROR}/package={pack}'
+ assert artifact_url == f'{CRAN_MIRROR}/src/contrib/{pack}_{vers}.tar.gz'
-def test_cran_compute_package_url_failure():
+def test_cran_compute_origin_urls_failure():
for incomplete_repo in [{'Version': '0.0.1'}, {'Package': 'package'}, {}]:
with pytest.raises(KeyError):
- compute_package_url(incomplete_repo)
+ compute_origin_urls(incomplete_repo)
@patch('swh.lister.cran.lister.read_cran_data')
@@ -51,7 +54,12 @@
# kwargs
kwargs = row['arguments']['kwargs']
assert len(kwargs) == 2
- assert set(kwargs.keys()) == {'url', 'version'}
+ assert set(kwargs.keys()) == {'url', 'artifacts'}
+
+ artifacts = kwargs['artifacts']
+ assert len(artifacts) == 1
+
+ assert set(artifacts[0].keys()) == {'url', 'version'}
assert row['policy'] == 'oneshot'
assert row['retries_left'] == 3

File Metadata

Mime Type
text/plain
Expires
Jul 3 2025, 9:50 AM (5 w, 3 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3221797

Event Timeline

About · Developer information · Legal