Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/pubdev/lister.py
| # Copyright (C) 2022 The Software Heritage developers | # Copyright (C) 2022 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import logging | import logging | ||||
| from typing import Any, Dict, Iterator, List, Optional | from typing import Any, Dict, Iterator, List, Optional | ||||
| import iso8601 | import iso8601 | ||||
| import requests | import requests | ||||
| from requests.exceptions import HTTPError | from requests.exceptions import HTTPError | ||||
| from tenacity.before_sleep import before_sleep_log | from tenacity.before_sleep import before_sleep_log | ||||
| from swh.lister.utils import throttling_retry | from swh.lister.utils import http_retry | ||||
| from swh.scheduler.interface import SchedulerInterface | from swh.scheduler.interface import SchedulerInterface | ||||
| from swh.scheduler.model import ListedOrigin | from swh.scheduler.model import ListedOrigin | ||||
| from .. import __version__ | from .. import __version__ | ||||
| from ..pattern import CredentialsType, StatelessLister | from ..pattern import CredentialsType, StatelessLister | ||||
| # https://github.com/dart-lang/pub/blob/master/doc/repository-spec-v2.md#metadata-headers | # https://github.com/dart-lang/pub/blob/master/doc/repository-spec-v2.md#metadata-headers | ||||
| USER_AGENT = ( | USER_AGENT = ( | ||||
| Show All 33 Lines | ): | ||||
| self.session = requests.Session() | self.session = requests.Session() | ||||
| self.session.headers.update( | self.session.headers.update( | ||||
| { | { | ||||
| "Accept": "application/json", | "Accept": "application/json", | ||||
| "User-Agent": USER_AGENT, | "User-Agent": USER_AGENT, | ||||
| } | } | ||||
| ) | ) | ||||
| @throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING)) | @http_retry(before_sleep=before_sleep_log(logger, logging.WARNING)) | ||||
| def page_request(self, url: str, params: Dict[str, Any]) -> requests.Response: | def page_request(self, url: str, params: Dict[str, Any]) -> requests.Response: | ||||
| logger.debug("Fetching URL %s with params %s", url, params) | logger.debug("Fetching URL %s with params %s", url, params) | ||||
| response = self.session.get(url, params=params) | response = self.session.get(url, params=params) | ||||
| if response.status_code != 200: | if response.status_code != 200: | ||||
| logger.warning( | logger.warning( | ||||
| "Unexpected HTTP status code %s on %s: %s", | "Unexpected HTTP status code %s on %s: %s", | ||||
| ▲ Show 20 Lines • Show All 54 Lines • Show Last 20 Lines | |||||