Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/phabricator/lister.py
# Copyright (C) 2019-2021 the Software Heritage developers | # Copyright (C) 2019-2022 the Software Heritage developers | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from collections import defaultdict | from collections import defaultdict | ||||
import logging | import logging | ||||
import random | import random | ||||
from typing import Any, Dict, Iterator, List, Optional | from typing import Any, Dict, Iterator, List, Optional | ||||
from urllib.parse import urljoin | from urllib.parse import urljoin | ||||
import requests | |||||
from swh.lister import USER_AGENT | |||||
from swh.lister.pattern import CredentialsType, StatelessLister | from swh.lister.pattern import CredentialsType, StatelessLister | ||||
from swh.scheduler.interface import SchedulerInterface | from swh.scheduler.interface import SchedulerInterface | ||||
from swh.scheduler.model import ListedOrigin | from swh.scheduler.model import ListedOrigin | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
PageType = List[Dict[str, Any]] | PageType = List[Dict[str, Any]] | ||||
Show All 21 Lines | def __init__( | ||||
instance: Optional[str] = None, | instance: Optional[str] = None, | ||||
api_token: Optional[str] = None, | api_token: Optional[str] = None, | ||||
credentials: CredentialsType = None, | credentials: CredentialsType = None, | ||||
): | ): | ||||
super().__init__( | super().__init__( | ||||
scheduler, urljoin(url, self.API_REPOSITORY_PATH), instance, credentials | scheduler, urljoin(url, self.API_REPOSITORY_PATH), instance, credentials | ||||
) | ) | ||||
self.session = requests.Session() | self.session.headers.update({"Accept": "application/json"}) | ||||
self.session.headers.update( | |||||
{"Accept": "application/json", "User-Agent": USER_AGENT} | |||||
) | |||||
if api_token is not None: | if api_token is not None: | ||||
self.api_token = api_token | self.api_token = api_token | ||||
else: | else: | ||||
if not self.credentials: | if not self.credentials: | ||||
raise ValueError( | raise ValueError( | ||||
f"No credentials found for phabricator instance {self.instance};" | f"No credentials found for phabricator instance {self.instance};" | ||||
" Please set them in the lister configuration file." | " Please set them in the lister configuration file." | ||||
Show All 24 Lines | def filter_params(params: Dict[str, str]) -> Dict[str, str]: | ||||
return { | return { | ||||
k: (v if k != "api.token" else "**redacted**") for k, v in params.items() | k: (v if k != "api.token" else "**redacted**") for k, v in params.items() | ||||
} | } | ||||
def get_pages(self) -> Iterator[PageType]: | def get_pages(self) -> Iterator[PageType]: | ||||
after: Optional[str] = None | after: Optional[str] = None | ||||
while True: | while True: | ||||
params = self.get_request_params(after) | params = self.get_request_params(after) | ||||
logger.debug( | response = self.http_request(self.url, method="POST", data=params) | ||||
"Retrieving results on URI %s with parameters %s", | |||||
self.url, | |||||
self.filter_params(params), | |||||
) | |||||
response = self.session.post(self.url, data=params) | |||||
if response.status_code != 200: | |||||
logger.warning( | |||||
"Unexpected HTTP status code %s on %s: %s", | |||||
response.status_code, | |||||
response.url, | |||||
response.content, | |||||
) | |||||
response.raise_for_status() | |||||
response_data = response.json() | response_data = response.json() | ||||
if response_data.get("result") is None: | if response_data.get("result") is None: | ||||
logger.warning( | logger.warning( | ||||
"Got unexpected response on %s: %s", | "Got unexpected response on %s: %s", | ||||
response.url, | response.url, | ||||
response_data, | response_data, | ||||
▲ Show 20 Lines • Show All 68 Lines • Show Last 20 Lines |