diff --git a/swh/lister/gitlab/lister.py b/swh/lister/gitlab/lister.py --- a/swh/lister/gitlab/lister.py +++ b/swh/lister/gitlab/lister.py @@ -5,6 +5,7 @@ from dataclasses import asdict, dataclass import logging +import random from typing import Any, Dict, Iterator, Optional, Tuple from urllib.parse import parse_qs, urlparse @@ -93,7 +94,7 @@ def __init__( self, scheduler, - url=None, + url: str, instance: Optional[str] = None, credentials: Optional[CredentialsType] = None, incremental: bool = False, @@ -101,10 +102,7 @@ if instance is None: instance = parse_url(url).host super().__init__( - scheduler=scheduler, - credentials=None, # anonymous for now - url=url, - instance=instance, + scheduler=scheduler, url=url, instance=instance, credentials=credentials, ) self.incremental = incremental self.last_page: Optional[str] = None @@ -114,12 +112,24 @@ {"Accept": "application/json", "User-Agent": USER_AGENT} ) + if len(self.credentials) > 0: + cred = random.choice(self.credentials) + logger.warning( + "Using %s credentials from user %s", self.instance, cred["username"] + ) + self.set_credentials(cred["username"], cred["password"]) + def state_from_dict(self, d: Dict[str, Any]) -> GitLabListerState: return GitLabListerState(**d) def state_to_dict(self, state: GitLabListerState) -> Dict[str, Any]: return asdict(state) + def set_credentials(self, username: Optional[str], password: Optional[str]) -> None: + """Set basic authentication headers with given credentials.""" + if username is not None and password is not None: + self.session.auth = (username, password) + @throttling_retry( retry=_if_rate_limited, before_sleep=before_sleep_log(logger, logging.WARNING) ) diff --git a/swh/lister/gitlab/tests/test_lister.py b/swh/lister/gitlab/tests/test_lister.py --- a/swh/lister/gitlab/tests/test_lister.py +++ b/swh/lister/gitlab/tests/test_lister.py @@ -198,6 +198,19 @@ assert_sleep_calls(mocker, mock_sleep, [1, WAIT_EXP_BASE]) +def test_lister_gitlab_credentials(swh_scheduler): + """Gitlab lister supports credentials configuration + + """ + instance = "gitlab" + credentials = {"gitlab": {instance: [{"username": "u", "password": "p"}]}} + url = api_url(instance) + lister = GitLabLister( + scheduler=swh_scheduler, url=url, instance=instance, credentials=credentials + ) + assert lister.session.auth == ("u", "p") + + @pytest.mark.parametrize( "url,expected_result", [