Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/pattern.py
Show All 12 Lines | |||||
import requests | import requests | ||||
from tenacity.before_sleep import before_sleep_log | from tenacity.before_sleep import before_sleep_log | ||||
from swh.core.config import load_from_envvar | from swh.core.config import load_from_envvar | ||||
from swh.core.utils import grouper | from swh.core.utils import grouper | ||||
from swh.scheduler import get_scheduler, model | from swh.scheduler import get_scheduler, model | ||||
from swh.scheduler.interface import SchedulerInterface | from swh.scheduler.interface import SchedulerInterface | ||||
from . import USER_AGENT | from . import USER_AGENT_TEMPLATE | ||||
from .utils import http_retry | from .utils import http_retry | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
@dataclass | @dataclass | ||||
class ListerStats: | class ListerStats: | ||||
pages: int = 0 | pages: int = 0 | ||||
▲ Show 20 Lines • Show All 89 Lines • ▼ Show 20 Lines | ): | ||||
) | ) | ||||
# store the initial state of the lister | # store the initial state of the lister | ||||
self.state = self.get_state_from_scheduler() | self.state = self.get_state_from_scheduler() | ||||
self.updated = False | self.updated = False | ||||
self.session = requests.Session() | self.session = requests.Session() | ||||
# Declare the USER_AGENT is more sysadm-friendly for the forge we list | # Declare the USER_AGENT is more sysadm-friendly for the forge we list | ||||
self.session.headers.update({"User-Agent": USER_AGENT}) | self.session.headers.update( | ||||
{"User-Agent": USER_AGENT_TEMPLATE % self.LISTER_NAME} | |||||
) | |||||
@http_retry(before_sleep=before_sleep_log(logger, logging.WARNING)) | @http_retry(before_sleep=before_sleep_log(logger, logging.WARNING)) | ||||
def http_request(self, url: str, method="GET", **kwargs) -> requests.Response: | def http_request(self, url: str, method="GET", **kwargs) -> requests.Response: | ||||
logger.debug("Fetching URL %s with params %s", url, kwargs.get("params")) | logger.debug("Fetching URL %s with params %s", url, kwargs.get("params")) | ||||
response = self.session.request(method, url, **kwargs) | response = self.session.request(method, url, **kwargs) | ||||
if response.status_code not in (200, 304): | if response.status_code not in (200, 304): | ||||
▲ Show 20 Lines • Show All 179 Lines • Show Last 20 Lines |