Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/gitlab/lister.py
| # Copyright (C) 2018-2021 The Software Heritage developers | # Copyright (C) 2018-2022 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from dataclasses import asdict, dataclass | from dataclasses import asdict, dataclass | ||||
| import logging | import logging | ||||
| import random | import random | ||||
| from typing import Any, Dict, Iterator, Optional, Tuple | from typing import Any, Dict, Iterator, Optional, Tuple | ||||
| from urllib.parse import parse_qs, urlencode, urlparse | from urllib.parse import parse_qs, urlencode, urlparse | ||||
| import iso8601 | import iso8601 | ||||
| import requests | |||||
| from requests.exceptions import HTTPError | from requests.exceptions import HTTPError | ||||
| from requests.status_codes import codes | from requests.status_codes import codes | ||||
| from tenacity.before_sleep import before_sleep_log | from tenacity.before_sleep import before_sleep_log | ||||
| from swh.lister import USER_AGENT | |||||
| from swh.lister.pattern import CredentialsType, Lister | from swh.lister.pattern import CredentialsType, Lister | ||||
| from swh.lister.utils import http_retry, is_retryable_exception | from swh.lister.utils import http_retry, is_retryable_exception | ||||
| from swh.scheduler.model import ListedOrigin | from swh.scheduler.model import ListedOrigin | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| # Some instance provides hg_git type which can be ingested as hg origins | # Some instance provides hg_git type which can be ingested as hg origins | ||||
| ▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines | ): | ||||
| url=url.rstrip("/"), | url=url.rstrip("/"), | ||||
| instance=instance, | instance=instance, | ||||
| credentials=credentials, | credentials=credentials, | ||||
| ) | ) | ||||
| self.incremental = incremental | self.incremental = incremental | ||||
| self.last_page: Optional[str] = None | self.last_page: Optional[str] = None | ||||
| self.per_page = 100 | self.per_page = 100 | ||||
| self.session = requests.Session() | self.session.headers.update({"Accept": "application/json"}) | ||||
| self.session.headers.update( | |||||
| {"Accept": "application/json", "User-Agent": USER_AGENT} | |||||
| ) | |||||
| if len(self.credentials) > 0: | if len(self.credentials) > 0: | ||||
| cred = random.choice(self.credentials) | cred = random.choice(self.credentials) | ||||
| logger.info( | logger.info( | ||||
| "Using %s credentials from user %s", self.instance, cred["username"] | "Using %s credentials from user %s", self.instance, cred["username"] | ||||
| ) | ) | ||||
| api_token = cred["password"] | api_token = cred["password"] | ||||
| if api_token: | if api_token: | ||||
| ▲ Show 20 Lines • Show All 133 Lines • Show Last 20 Lines | |||||