diff --git a/swh/lister/debian/lister.py b/swh/lister/debian/lister.py --- a/swh/lister/debian/lister.py +++ b/swh/lister/debian/lister.py @@ -134,19 +134,17 @@ response = requests.get(url, stream=True) logging.debug("Fetched URL: %s, status code: %s", url, response.status_code) if response.status_code == 200: + decompressor = decompressors.get(compression) + if decompressor: + data = decompressor(response.raw).readlines() + else: + data = response.raw.readlines() break else: - raise Exception( - "Could not retrieve sources index for %s/%s", suite, component - ) - - decompressor = decompressors.get(compression) - if decompressor: - data = decompressor(response.raw) - else: - data = response.raw + data = "" + logger.debug("Could not retrieve sources index for %s/%s", suite, component) - return Sources.iter_paragraphs(data.readlines()) + return Sources.iter_paragraphs(data) def get_pages(self) -> Iterator[DebianPageType]: """Return an iterator on parsed debian package Sources files, one per combination diff --git a/swh/lister/debian/tests/test_lister.py b/swh/lister/debian/tests/test_lister.py --- a/swh/lister/debian/tests/test_lister.py +++ b/swh/lister/debian/tests/test_lister.py @@ -36,7 +36,7 @@ _mirror_url = "http://deb.debian.org/debian" _suites = ["stretch", "buster", "bullseye"] -_components = ["main"] +_components = ["main", "foo"] SourcesText = str @@ -59,6 +59,7 @@ debian_sources: Dict[Suite, SourcesText], requests_mock, ) -> Tuple[DebianLister, DebianSuitePkgSrcInfo]: + lister = DebianLister( scheduler=swh_scheduler, mirror_url=_mirror_url, @@ -79,6 +80,9 @@ else: requests_mock.get(idx_url, text=sources) + for idx_url, _ in lister.debian_index_urls(suite, _components[1]): + requests_mock.get(idx_url, status_code=404) + return lister, suite_pkg_info @@ -198,7 +202,7 @@ lister_previous_state=lister_previous_state, ) - assert stats.pages == len(sources) + assert stats.pages == len(sources) * len(_components) assert stats.origins == len(origin_urls) lister_previous_state = lister.state.package_versions