diff --git a/swh/lister/launchpad/lister.py b/swh/lister/launchpad/lister.py --- a/swh/lister/launchpad/lister.py +++ b/swh/lister/launchpad/lister.py @@ -134,6 +134,21 @@ continue yield vcs_type, page_result + def _read_page(self, page: LaunchpadPageType) -> Iterator[Tuple[str, Any]]: + """Read page of results and yield the vcs_type and associated repo. + + This indirection method exists as communication failures can still occur when + reading the page. This traps the error, logs and continues listing the next + pages. + + """ + try: + vcs_type, repos = page + for repo in repos: + yield vcs_type, repo + except RestfulError as e: + logger.error("Issue when reading %s repos %s: %s", vcs_type, repos, e) + def get_origins_from_page(self, page: LaunchpadPageType) -> Iterator[ListedOrigin]: """ Iterate on all git repositories and yield ListedOrigin instances. @@ -142,11 +157,7 @@ prev_origin_url: Dict[str, Optional[str]] = {"git": None, "bzr": None} - vcs_type, repos = page - - assert vcs_type in {"git", "bzr"} - - for repo in repos: + for vcs_type, repo in self._read_page(page): origin_url = origin(vcs_type, repo) # filter out origins with invalid URL or origin previously listed