Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/pubdev/lister.py
# Copyright (C) 2022 The Software Heritage developers | # Copyright (C) 2022 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import logging | import logging | ||||
from typing import Iterator, List, Optional | from typing import Iterator, List, Optional | ||||
import iso8601 | import iso8601 | ||||
from requests.exceptions import HTTPError | from requests.exceptions import HTTPError | ||||
from swh.scheduler.interface import SchedulerInterface | from swh.scheduler.interface import SchedulerInterface | ||||
from swh.scheduler.model import ListedOrigin | from swh.scheduler.model import ListedOrigin | ||||
from .. import __version__ | |||||
from ..pattern import CredentialsType, StatelessLister | from ..pattern import CredentialsType, StatelessLister | ||||
# https://github.com/dart-lang/pub/blob/master/doc/repository-spec-v2.md#metadata-headers | |||||
USER_AGENT = ( | |||||
f"Software Heritage PubDev Lister v{__version__} " | |||||
"(+https://www.softwareheritage.org/contact)" | |||||
) | |||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
# Aliasing the page results returned by `get_pages` method from the lister. | # Aliasing the page results returned by `get_pages` method from the lister. | ||||
PubDevListerPage = List[str] | PubDevListerPage = List[str] | ||||
class PubDevLister(StatelessLister[PubDevListerPage]): | class PubDevLister(StatelessLister[PubDevListerPage]): | ||||
"""List pub.dev (Dart, Flutter) origins.""" | """List pub.dev (Dart, Flutter) origins.""" | ||||
Show All 14 Lines | class PubDevLister(StatelessLister[PubDevListerPage]): | ||||
): | ): | ||||
super().__init__( | super().__init__( | ||||
scheduler=scheduler, | scheduler=scheduler, | ||||
credentials=credentials, | credentials=credentials, | ||||
instance=self.INSTANCE, | instance=self.INSTANCE, | ||||
url=self.BASE_URL, | url=self.BASE_URL, | ||||
) | ) | ||||
self.session.headers.update( | self.session.headers.update({"Accept": "application/json"}) | ||||
{ | |||||
"Accept": "application/json", | |||||
"User-Agent": USER_AGENT, | |||||
} | |||||
) | |||||
def get_pages(self) -> Iterator[PubDevListerPage]: | def get_pages(self) -> Iterator[PubDevListerPage]: | ||||
"""Yield an iterator which returns 'page' | """Yield an iterator which returns 'page' | ||||
It uses the api provided by https://pub.dev/api/ to find Dart and Flutter package | It uses the api provided by https://pub.dev/api/ to find Dart and Flutter package | ||||
origins. | origins. | ||||
The http api call get "{base_url}package-names" to retrieve a sorted list | The http api call get "{base_url}package-names" to retrieve a sorted list | ||||
Show All 39 Lines |