Differential D8523 Diff 30693 swh/lister/pubdev/lister.py

Changeset View

Standalone View

swh/lister/pubdev/lister.py

# Copyright (C) 2022 The Software Heritage developers		# Copyright (C) 2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import logging		import logging
from typing import Iterator, List, Optional		from typing import Iterator, List, Optional

import iso8601		import iso8601
from requests.exceptions import HTTPError		from requests.exceptions import HTTPError

from swh.scheduler.interface import SchedulerInterface		from swh.scheduler.interface import SchedulerInterface
from swh.scheduler.model import ListedOrigin		from swh.scheduler.model import ListedOrigin

from .. import __version__
from ..pattern import CredentialsType, StatelessLister		from ..pattern import CredentialsType, StatelessLister

# https://github.com/dart-lang/pub/blob/master/doc/repository-spec-v2.md#metadata-headers
USER_AGENT = (
f"Software Heritage PubDev Lister v{__version__} "
"(+https://www.softwareheritage.org/contact)"
)

logger = logging.getLogger(__name__)		logger = logging.getLogger(__name__)

# Aliasing the page results returned by `get_pages` method from the lister.		# Aliasing the page results returned by `get_pages` method from the lister.
PubDevListerPage = List[str]		PubDevListerPage = List[str]


class PubDevLister(StatelessLister[PubDevListerPage]):		class PubDevLister(StatelessLister[PubDevListerPage]):
"""List pub.dev (Dart, Flutter) origins."""		"""List pub.dev (Dart, Flutter) origins."""
Show All 14 Lines	class PubDevLister(StatelessLister[PubDevListerPage]):
):		):
super().__init__(		super().__init__(
scheduler=scheduler,		scheduler=scheduler,
credentials=credentials,		credentials=credentials,
instance=self.INSTANCE,		instance=self.INSTANCE,
url=self.BASE_URL,		url=self.BASE_URL,
)		)

self.session.headers.update(		self.session.headers.update({"Accept": "application/json"})
{
"Accept": "application/json",
"User-Agent": USER_AGENT,
}
)

def get_pages(self) -> Iterator[PubDevListerPage]:		def get_pages(self) -> Iterator[PubDevListerPage]:
"""Yield an iterator which returns 'page'		"""Yield an iterator which returns 'page'

It uses the api provided by https://pub.dev/api/ to find Dart and Flutter package		It uses the api provided by https://pub.dev/api/ to find Dart and Flutter package
origins.		origins.

The http api call get "{base_url}package-names" to retrieve a sorted list		The http api call get "{base_url}package-names" to retrieve a sorted list
Show All 39 Lines