Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/bitbucket/lister.py
# Copyright (C) 2017-2021 The Software Heritage developers | # Copyright (C) 2017-2021 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from dataclasses import asdict, dataclass | from dataclasses import asdict, dataclass | ||||
from datetime import datetime | from datetime import datetime | ||||
import logging | import logging | ||||
import random | import random | ||||
from typing import Any, Dict, Iterator, List, Optional | from typing import Any, Dict, Iterator, List, Optional | ||||
from urllib import parse | from urllib import parse | ||||
import iso8601 | import iso8601 | ||||
import requests | import requests | ||||
from tenacity.before_sleep import before_sleep_log | from tenacity.before_sleep import before_sleep_log | ||||
from swh.lister.utils import throttling_retry | from swh.lister.utils import http_retry | ||||
from swh.scheduler.interface import SchedulerInterface | from swh.scheduler.interface import SchedulerInterface | ||||
from swh.scheduler.model import ListedOrigin | from swh.scheduler.model import ListedOrigin | ||||
from .. import USER_AGENT | from .. import USER_AGENT | ||||
from ..pattern import CredentialsType, Lister | from ..pattern import CredentialsType, Lister | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines | def state_to_dict(self, state: BitbucketListerState) -> Dict[str, Any]: | ||||
d["last_repo_cdate"] = last_repo_cdate.isoformat() | d["last_repo_cdate"] = last_repo_cdate.isoformat() | ||||
return d | return d | ||||
def set_credentials(self, username: Optional[str], password: Optional[str]) -> None: | def set_credentials(self, username: Optional[str], password: Optional[str]) -> None: | ||||
"""Set basic authentication headers with given credentials.""" | """Set basic authentication headers with given credentials.""" | ||||
if username is not None and password is not None: | if username is not None and password is not None: | ||||
self.session.auth = (username, password) | self.session.auth = (username, password) | ||||
@throttling_retry(before_sleep=before_sleep_log(logger, logging.DEBUG)) | @http_retry(before_sleep=before_sleep_log(logger, logging.DEBUG)) | ||||
def page_request(self, last_repo_cdate: str) -> requests.Response: | def page_request(self, last_repo_cdate: str) -> requests.Response: | ||||
self.url_params["after"] = last_repo_cdate | self.url_params["after"] = last_repo_cdate | ||||
logger.debug("Fetching URL %s with params %s", self.url, self.url_params) | logger.debug("Fetching URL %s with params %s", self.url, self.url_params) | ||||
response = self.session.get(self.url, params=self.url_params) | response = self.session.get(self.url, params=self.url_params) | ||||
if response.status_code != 200: | if response.status_code != 200: | ||||
▲ Show 20 Lines • Show All 80 Lines • Show Last 20 Lines |