Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/arch/lister.py
Show All 10 Lines | |||||
import tempfile | import tempfile | ||||
from typing import Any, Dict, Iterator, List, Optional | from typing import Any, Dict, Iterator, List, Optional | ||||
from urllib.parse import unquote, urljoin | from urllib.parse import unquote, urljoin | ||||
from bs4 import BeautifulSoup | from bs4 import BeautifulSoup | ||||
import requests | import requests | ||||
from tenacity.before_sleep import before_sleep_log | from tenacity.before_sleep import before_sleep_log | ||||
from swh.lister.utils import throttling_retry | from swh.lister.utils import http_retry | ||||
from swh.model.hashutil import hash_to_hex | from swh.model.hashutil import hash_to_hex | ||||
from swh.scheduler.interface import SchedulerInterface | from swh.scheduler.interface import SchedulerInterface | ||||
from swh.scheduler.model import ListedOrigin | from swh.scheduler.model import ListedOrigin | ||||
from .. import USER_AGENT | from .. import USER_AGENT | ||||
from ..pattern import CredentialsType, StatelessLister | from ..pattern import CredentialsType, StatelessLister | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
▲ Show 20 Lines • Show All 99 Lines • ▼ Show 20 Lines | ): | ||||
self.flavours = flavours | self.flavours = flavours | ||||
self.session = requests.Session() | self.session = requests.Session() | ||||
self.session.headers.update( | self.session.headers.update( | ||||
{ | { | ||||
"User-Agent": USER_AGENT, | "User-Agent": USER_AGENT, | ||||
} | } | ||||
) | ) | ||||
@throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING)) | @http_retry(before_sleep=before_sleep_log(logger, logging.WARNING)) | ||||
def request_get(self, url: str, params: Dict[str, Any]) -> requests.Response: | def request_get(self, url: str, params: Dict[str, Any]) -> requests.Response: | ||||
logger.debug("Fetching URL %s with params %s", url, params) | logger.debug("Fetching URL %s with params %s", url, params) | ||||
response = self.session.get(url, params=params) | response = self.session.get(url, params=params) | ||||
if response.status_code != 200: | if response.status_code != 200: | ||||
logger.warning( | logger.warning( | ||||
"Unexpected HTTP status code %s on %s: %s", | "Unexpected HTTP status code %s on %s: %s", | ||||
▲ Show 20 Lines • Show All 358 Lines • Show Last 20 Lines |