Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/gitlab/lister.py
| Show All 11 Lines | |||||
| import iso8601 | import iso8601 | ||||
| import requests | import requests | ||||
| from requests.exceptions import HTTPError | from requests.exceptions import HTTPError | ||||
| from requests.status_codes import codes | from requests.status_codes import codes | ||||
| from tenacity.before_sleep import before_sleep_log | from tenacity.before_sleep import before_sleep_log | ||||
| from swh.lister import USER_AGENT | from swh.lister import USER_AGENT | ||||
| from swh.lister.pattern import CredentialsType, Lister | from swh.lister.pattern import CredentialsType, Lister | ||||
| from swh.lister.utils import is_retryable_exception, throttling_retry | from swh.lister.utils import http_retry, is_retryable_exception | ||||
| from swh.scheduler.model import ListedOrigin | from swh.scheduler.model import ListedOrigin | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| # Some instance provides hg_git type which can be ingested as hg origins | # Some instance provides hg_git type which can be ingested as hg origins | ||||
| VCS_MAPPING = {"hg_git": "hg"} | VCS_MAPPING = {"hg_git": "hg"} | ||||
| ▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines | ): | ||||
| self.session.headers["Authorization"] = f"Bearer {api_token}" | self.session.headers["Authorization"] = f"Bearer {api_token}" | ||||
| def state_from_dict(self, d: Dict[str, Any]) -> GitLabListerState: | def state_from_dict(self, d: Dict[str, Any]) -> GitLabListerState: | ||||
| return GitLabListerState(**d) | return GitLabListerState(**d) | ||||
| def state_to_dict(self, state: GitLabListerState) -> Dict[str, Any]: | def state_to_dict(self, state: GitLabListerState) -> Dict[str, Any]: | ||||
| return asdict(state) | return asdict(state) | ||||
| @throttling_retry( | @http_retry( | ||||
| retry=_if_rate_limited, before_sleep=before_sleep_log(logger, logging.WARNING) | retry=_if_rate_limited, before_sleep=before_sleep_log(logger, logging.WARNING) | ||||
| ) | ) | ||||
| def get_page_result(self, url: str) -> PageResult: | def get_page_result(self, url: str) -> PageResult: | ||||
| logger.debug("Fetching URL %s", url) | logger.debug("Fetching URL %s", url) | ||||
| response = self.session.get(url) | response = self.session.get(url) | ||||
| if response.status_code != 200: | if response.status_code != 200: | ||||
| logger.warning( | logger.warning( | ||||
| "Unexpected HTTP status code %s on %s: %s", | "Unexpected HTTP status code %s on %s: %s", | ||||
| ▲ Show 20 Lines • Show All 116 Lines • Show Last 20 Lines | |||||