Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/npm/lister.py
| # Copyright (C) 2018-2021 the Software Heritage developers | # Copyright (C) 2018-2021 the Software Heritage developers | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from dataclasses import asdict, dataclass | from dataclasses import asdict, dataclass | ||||
| import logging | import logging | ||||
| from typing import Any, Dict, Iterator, List, Optional | from typing import Any, Dict, Iterator, List, Optional | ||||
| import iso8601 | import iso8601 | ||||
| import requests | import requests | ||||
| from tenacity.before_sleep import before_sleep_log | from tenacity.before_sleep import before_sleep_log | ||||
| from swh.lister import USER_AGENT | from swh.lister import USER_AGENT | ||||
| from swh.lister.pattern import CredentialsType, Lister | from swh.lister.pattern import CredentialsType, Lister | ||||
| from swh.lister.utils import throttling_retry | from swh.lister.utils import http_retry | ||||
| from swh.scheduler.interface import SchedulerInterface | from swh.scheduler.interface import SchedulerInterface | ||||
| from swh.scheduler.model import ListedOrigin | from swh.scheduler.model import ListedOrigin | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| @dataclass | @dataclass | ||||
| class NpmListerState: | class NpmListerState: | ||||
| ▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | def request_params(self, last_package_id: str) -> Dict[str, Any]: | ||||
| # include package JSON document to get its last update date | # include package JSON document to get its last update date | ||||
| params = {"limit": self.page_size, "include_docs": "true"} | params = {"limit": self.page_size, "include_docs": "true"} | ||||
| if self.incremental: | if self.incremental: | ||||
| params["since"] = last_package_id | params["since"] = last_package_id | ||||
| else: | else: | ||||
| params["startkey"] = last_package_id | params["startkey"] = last_package_id | ||||
| return params | return params | ||||
| @throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING)) | @http_retry(before_sleep=before_sleep_log(logger, logging.WARNING)) | ||||
| def page_request(self, last_package_id: str) -> requests.Response: | def page_request(self, last_package_id: str) -> requests.Response: | ||||
| params = self.request_params(last_package_id) | params = self.request_params(last_package_id) | ||||
| logger.debug("Fetching URL %s with params %s", self.url, params) | logger.debug("Fetching URL %s with params %s", self.url, params) | ||||
| response = self.session.get(self.url, params=params) | response = self.session.get(self.url, params=params) | ||||
| if response.status_code != 200: | if response.status_code != 200: | ||||
| logger.warning( | logger.warning( | ||||
| "Unexpected HTTP status code %s on %s: %s", | "Unexpected HTTP status code %s on %s: %s", | ||||
| response.status_code, | response.status_code, | ||||
| ▲ Show 20 Lines • Show All 84 Lines • Show Last 20 Lines | |||||