Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/aur/lister.py
| # Copyright (C) 2022 The Software Heritage developers | # Copyright (C) 2022 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import datetime | import datetime | ||||
| import logging | import logging | ||||
| from typing import Any, Dict, Iterator, List, Optional | from typing import Any, Dict, Iterator, List, Optional | ||||
| import requests | |||||
| from swh.scheduler.interface import SchedulerInterface | from swh.scheduler.interface import SchedulerInterface | ||||
| from swh.scheduler.model import ListedOrigin | from swh.scheduler.model import ListedOrigin | ||||
| from ..pattern import CredentialsType, StatelessLister | from ..pattern import CredentialsType, StatelessLister | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| # Aliasing the page results returned by `get_pages` method from the lister. | # Aliasing the page results returned by `get_pages` method from the lister. | ||||
| Show All 40 Lines | class AurLister(StatelessLister[AurListerPage]): | ||||
| def download_packages_index(self) -> List[Dict[str, Any]]: | def download_packages_index(self) -> List[Dict[str, Any]]: | ||||
| """Build an url based on self.DEFAULT_PACKAGES_INDEX_URL format string, | """Build an url based on self.DEFAULT_PACKAGES_INDEX_URL format string, | ||||
| and download the archive to self.DESTINATION_PATH | and download the archive to self.DESTINATION_PATH | ||||
| Returns: | Returns: | ||||
| a directory Path where the archive has been downloaded to. | a directory Path where the archive has been downloaded to. | ||||
| """ | """ | ||||
| url = self.DEFAULT_PACKAGES_INDEX_URL.format(base_url=self.url) | url = self.DEFAULT_PACKAGES_INDEX_URL.format(base_url=self.url) | ||||
| return requests.get(url).json() | return self.http_request(url).json() | ||||
| def get_pages(self) -> Iterator[AurListerPage]: | def get_pages(self) -> Iterator[AurListerPage]: | ||||
| """Yield an iterator which returns 'page' | """Yield an iterator which returns 'page' | ||||
| Each page corresponds to a package with a 'version', an 'url' for a Git | Each page corresponds to a package with a 'version', an 'url' for a Git | ||||
| repository, a 'project_url' which represents the upstream project url and | repository, a 'project_url' which represents the upstream project url and | ||||
| a canonical 'snapshot_url' from which a tar.gz archive of the package can | a canonical 'snapshot_url' from which a tar.gz archive of the package can | ||||
| be downloaded. | be downloaded. | ||||
| ▲ Show 20 Lines • Show All 78 Lines • Show Last 20 Lines | |||||