diff --git a/swh/lister/launchpad/lister.py b/swh/lister/launchpad/lister.py --- a/swh/lister/launchpad/lister.py +++ b/swh/lister/launchpad/lister.py @@ -10,8 +10,11 @@ import iso8601 from launchpadlib.launchpad import Launchpad +from lazr.restfulclient.errors import RestfulError from lazr.restfulclient.resource import Collection +from tenacity.before_sleep import before_sleep_log +from swh.lister.utils import throttling_retry from swh.scheduler.interface import SchedulerInterface from swh.scheduler.model import ListedOrigin @@ -90,6 +93,30 @@ d[attribute_name] = date_last_modified.isoformat() return d + @throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING)) + def page_request( + self, launchpad, vcs_type: str, date_last_modified: datetime + ) -> Optional[Collection]: + """Querying the page of results for a given vcs_type since the date_last_modified. If + some issues occurs, this will deal with the retrying policy. + + """ + get_vcs_fns = { + "git": launchpad.git_repositories.getRepositories, + "bzr": launchpad.branches.getBranches, + } + + page_result = None + try: + page_result = get_vcs_fns[vcs_type]( + order_by="most neglected first", + modified_since_date=self.date_last_modified[vcs_type], + ) + except RestfulError as e: + logger.error("Listing %s origins raised %s", vcs_type, e) + + return page_result + def get_pages(self) -> Iterator[LaunchpadPageType]: """ Yields an iterator on all git/bzr repositories hosted on Launchpad sorted @@ -103,14 +130,13 @@ "git": self.state.git_date_last_modified, "bzr": self.state.bzr_date_last_modified, } - for vcs_type, get_vcs_fn in [ - ("git", launchpad.git_repositories.getRepositories), - ("bzr", launchpad.branches.getBranches), - ]: - yield vcs_type, get_vcs_fn( - order_by="most neglected first", - modified_since_date=self.date_last_modified[vcs_type], + for vcs_type in ["git", "bzr"]: + page_result = self.page_request( + launchpad, vcs_type, self.date_last_modified[vcs_type] ) + if not page_result: + continue + yield vcs_type, page_result def get_origins_from_page(self, page: LaunchpadPageType) -> Iterator[ListedOrigin]: """ diff --git a/swh/lister/launchpad/tests/test_lister.py b/swh/lister/launchpad/tests/test_lister.py --- a/swh/lister/launchpad/tests/test_lister.py +++ b/swh/lister/launchpad/tests/test_lister.py @@ -166,7 +166,7 @@ assert lister.incremental assert lister.updated - assert stats.pages == 2, "Empty bzr response still accounts for 1 page" + assert stats.pages == 1, "Empty bzr page response is ignored" assert stats.origins == len(launchpad_response2) mock_getRepositories.assert_called_once_with( @@ -192,7 +192,7 @@ stats = lister.run() assert not lister.updated - assert stats.pages == 1 + 1, "Empty pages are still accounted for (1 git, 1 bzr)" + assert stats.pages == 1, "Empty pages are ignored(only 1 git page of results)" assert stats.origins == 0 @@ -211,5 +211,5 @@ stats = lister.run() assert lister.updated - assert stats.pages == 1 + 1, "Empty bzr page is still accounted for (1 git, 1 bzr)" + assert stats.pages == 1, "Empty bzr page are ignored (only 1 git page of results)" assert stats.origins == 1