diff --git a/swh/lister/cgit/lister.py b/swh/lister/cgit/lister.py --- a/swh/lister/cgit/lister.py +++ b/swh/lister/cgit/lister.py @@ -13,7 +13,7 @@ from requests.exceptions import HTTPError from swh.lister import USER_AGENT -from swh.lister.pattern import StatelessLister +from swh.lister.pattern import CredentialsType, StatelessLister from swh.scheduler.interface import SchedulerInterface from swh.scheduler.model import ListedOrigin @@ -39,7 +39,11 @@ LISTER_NAME = "cgit" def __init__( - self, scheduler: SchedulerInterface, url: str, instance: Optional[str] = None + self, + scheduler: SchedulerInterface, + url: str, + instance: Optional[str] = None, + credentials: Optional[CredentialsType] = None, ): """Lister class for CGit repositories. @@ -55,7 +59,7 @@ assert instance is not None # Make mypy happy super().__init__( - scheduler=scheduler, credentials=None, url=url, instance=instance, + scheduler=scheduler, url=url, instance=instance, credentials=credentials, ) self.session = requests.Session() diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py --- a/swh/lister/cgit/tests/test_lister.py +++ b/swh/lister/cgit/tests/test_lister.py @@ -160,3 +160,39 @@ expected_nb_origins = 15 assert stats == ListerStats(pages=3, origins=expected_nb_origins) + + +@pytest.mark.parametrize( + "credentials, expected_credentials", + [ + (None, []), + ({"key": "value"}, []), + ( + {"cgit": {"tizen": [{"username": "user", "password": "pass"}]}}, + [{"username": "user", "password": "pass"}], + ), + ], +) +def test_lister_cgit_instantiation_with_credentials( + credentials, expected_credentials, swh_scheduler +): + url = "https://git.tizen/cgit/" + lister = CGitLister( + swh_scheduler, url=url, instance="tizen", credentials=credentials + ) + + # Credentials are allowed in constructor + assert lister.credentials == expected_credentials + + +def test_lister_cgit_from_configfile(swh_scheduler_config, mocker): + load_from_envvar = mocker.patch("swh.lister.pattern.load_from_envvar") + load_from_envvar.return_value = { + "scheduler": {"cls": "local", **swh_scheduler_config}, + "url": "https://git.tizen/cgit/", + "instance": "tizen", + "credentials": {}, + } + lister = CGitLister.from_configfile() + assert lister.scheduler is not None + assert lister.credentials is not None diff --git a/swh/lister/cgit/tests/test_tasks.py b/swh/lister/cgit/tests/test_tasks.py --- a/swh/lister/cgit/tests/test_tasks.py +++ b/swh/lister/cgit/tests/test_tasks.py @@ -3,8 +3,6 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from unittest.mock import patch - from swh.lister.pattern import ListerStats @@ -16,11 +14,11 @@ assert res.result == "OK" -@patch("swh.lister.cgit.tasks.CGitLister") def test_cgit_lister_task( - lister, swh_scheduler_celery_app, swh_scheduler_celery_worker + swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker ): # setup the mocked CGitLister + lister = mocker.patch("swh.lister.cgit.tasks.CGitLister") lister.from_configfile.return_value = lister lister.run.return_value = ListerStats(pages=10, origins=500) diff --git a/swh/lister/cran/lister.py b/swh/lister/cran/lister.py --- a/swh/lister/cran/lister.py +++ b/swh/lister/cran/lister.py @@ -10,7 +10,7 @@ import pkg_resources -from swh.lister.pattern import StatelessLister +from swh.lister.pattern import CredentialsType, StatelessLister from swh.scheduler.interface import SchedulerInterface from swh.scheduler.model import ListedOrigin @@ -29,9 +29,13 @@ LISTER_NAME = "CRAN" def __init__( - self, scheduler: SchedulerInterface, + self, + scheduler: SchedulerInterface, + credentials: Optional[CredentialsType] = None, ): - super().__init__(scheduler, url=CRAN_MIRROR, instance="cran") + super().__init__( + scheduler, url=CRAN_MIRROR, instance="cran", credentials=credentials + ) def get_pages(self) -> Iterator[PageType]: """ diff --git a/swh/lister/cran/tests/test_lister.py b/swh/lister/cran/tests/test_lister.py --- a/swh/lister/cran/tests/test_lister.py +++ b/swh/lister/cran/tests/test_lister.py @@ -89,3 +89,34 @@ } filtered_origins[0].last_update == parse_packaged_date(package_info) + + +@pytest.mark.parametrize( + "credentials, expected_credentials", + [ + (None, []), + ({"key": "value"}, []), + ( + {"CRAN": {"cran": [{"username": "user", "password": "pass"}]}}, + [{"username": "user", "password": "pass"}], + ), + ], +) +def test_lister_cran_instantiation_with_credentials( + credentials, expected_credentials, swh_scheduler +): + lister = CRANLister(swh_scheduler, credentials=credentials) + + # Credentials are allowed in constructor + assert lister.credentials == expected_credentials + + +def test_lister_cran_from_configfile(swh_scheduler_config, mocker): + load_from_envvar = mocker.patch("swh.lister.pattern.load_from_envvar") + load_from_envvar.return_value = { + "scheduler": {"cls": "local", **swh_scheduler_config}, + "credentials": {}, + } + lister = CRANLister.from_configfile() + assert lister.scheduler is not None + assert lister.credentials is not None diff --git a/swh/lister/pypi/lister.py b/swh/lister/pypi/lister.py --- a/swh/lister/pypi/lister.py +++ b/swh/lister/pypi/lister.py @@ -1,10 +1,10 @@ -# Copyright (C) 2018-2019 The Software Heritage developers +# Copyright (C) 2018-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging -from typing import Iterator, List +from typing import Iterator, List, Optional import requests import xmltodict @@ -13,7 +13,7 @@ from swh.scheduler.model import ListedOrigin from .. import USER_AGENT -from ..pattern import StatelessLister +from ..pattern import CredentialsType, StatelessLister logger = logging.getLogger(__name__) @@ -31,12 +31,16 @@ PACKAGE_LIST_URL = "https://pypi.org/simple/" PACKAGE_URL = "https://pypi.org/project/{package_name}/" - def __init__(self, scheduler: SchedulerInterface): + def __init__( + self, + scheduler: SchedulerInterface, + credentials: Optional[CredentialsType] = None, + ): super().__init__( scheduler=scheduler, - credentials=None, url=self.PACKAGE_LIST_URL, instance=self.INSTANCE, + credentials=credentials, ) self.session = requests.Session() diff --git a/swh/lister/pypi/tests/test_lister.py b/swh/lister/pypi/tests/test_lister.py --- a/swh/lister/pypi/tests/test_lister.py +++ b/swh/lister/pypi/tests/test_lister.py @@ -78,3 +78,34 @@ scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results assert len(scheduler_origins) == 0 + + +@pytest.mark.parametrize( + "credentials, expected_credentials", + [ + (None, []), + ({"key": "value"}, []), + ( + {"pypi": {"pypi": [{"username": "user", "password": "pass"}]}}, + [{"username": "user", "password": "pass"}], + ), + ], +) +def test_lister_pypi_instantiation_with_credentials( + credentials, expected_credentials, swh_scheduler +): + lister = PyPILister(swh_scheduler, credentials=credentials) + + # Credentials are allowed in constructor + assert lister.credentials == expected_credentials + + +def test_lister_pypi_from_configfile(swh_scheduler_config, mocker): + load_from_envvar = mocker.patch("swh.lister.pattern.load_from_envvar") + load_from_envvar.return_value = { + "scheduler": {"cls": "local", **swh_scheduler_config}, + "credentials": {}, + } + lister = PyPILister.from_configfile() + assert lister.scheduler is not None + assert lister.credentials is not None