Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/launchpad/lister.py
| # Copyright (C) 2020-2022 The Software Heritage developers | # Copyright (C) 2020-2022 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from dataclasses import dataclass | from dataclasses import dataclass | ||||
| from datetime import datetime | from datetime import datetime | ||||
| import logging | import logging | ||||
| from typing import Any, Dict, Iterator, Optional, Tuple | from typing import Any, Dict, Iterator, Optional, Tuple | ||||
| import iso8601 | import iso8601 | ||||
| from launchpadlib.launchpad import Launchpad | from launchpadlib.launchpad import Launchpad | ||||
| from lazr.restfulclient.errors import RestfulError | from lazr.restfulclient.errors import RestfulError | ||||
| from lazr.restfulclient.resource import Collection | from lazr.restfulclient.resource import Collection | ||||
| from tenacity.before_sleep import before_sleep_log | from tenacity.before_sleep import before_sleep_log | ||||
| from swh.lister.utils import retry_if_exception, throttling_retry | from swh.lister.utils import http_retry, retry_if_exception | ||||
| from swh.scheduler.interface import SchedulerInterface | from swh.scheduler.interface import SchedulerInterface | ||||
| from swh.scheduler.model import ListedOrigin | from swh.scheduler.model import ListedOrigin | ||||
| from ..pattern import CredentialsType, Lister | from ..pattern import CredentialsType, Lister | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| VcsType = str | VcsType = str | ||||
| ▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | def state_to_dict(self, state: LaunchpadListerState) -> Dict[str, Any]: | ||||
| d[attribute_name] = None | d[attribute_name] = None | ||||
| if hasattr(state, attribute_name): | if hasattr(state, attribute_name): | ||||
| date_last_modified = getattr(state, attribute_name) | date_last_modified = getattr(state, attribute_name) | ||||
| if date_last_modified is not None: | if date_last_modified is not None: | ||||
| d[attribute_name] = date_last_modified.isoformat() | d[attribute_name] = date_last_modified.isoformat() | ||||
| return d | return d | ||||
| @throttling_retry( | @http_retry( | ||||
| retry=retry_if_restful_error, | retry=retry_if_restful_error, | ||||
| before_sleep=before_sleep_log(logger, logging.WARNING), | before_sleep=before_sleep_log(logger, logging.WARNING), | ||||
| ) | ) | ||||
| def _page_request( | def _page_request( | ||||
| self, launchpad, vcs_type: str, date_last_modified: Optional[datetime] | self, launchpad, vcs_type: str, date_last_modified: Optional[datetime] | ||||
| ) -> Optional[Collection]: | ) -> Optional[Collection]: | ||||
| """Querying the page of results for a given vcs_type since the date_last_modified. If | """Querying the page of results for a given vcs_type since the date_last_modified. If | ||||
| some issues occurs, this will deal with the retrying policy. | some issues occurs, this will deal with the retrying policy. | ||||
| ▲ Show 20 Lines • Show All 98 Lines • Show Last 20 Lines | |||||