diff --git a/swh/lister/conda/lister.py b/swh/lister/conda/lister.py --- a/swh/lister/conda/lister.py +++ b/swh/lister/conda/lister.py @@ -71,7 +71,9 @@ assert self.lister_obj.id is not None arch, packages = page + package_names = set() for filename, package_metadata in packages.items(): + package_names.add(package_metadata["name"]) version_key = ( f"{arch}/{package_metadata['version']}-{package_metadata['build']}" ) @@ -102,22 +104,20 @@ elif "date" in package_metadata: package_date = iso8601.parse_date(package_metadata["date"]) - last_update = None if package_date: artifact["date"] = package_date.isoformat() self.package_dates[package_metadata["name"]].append(package_date) - last_update = max(self.package_dates[package_metadata["name"]]) + for package_name in package_names: + package_dates = self.package_dates[package_name] yield ListedOrigin( lister_id=self.lister_obj.id, visit_type=self.VISIT_TYPE, url=self.ORIGIN_URL_PATTERN.format( - channel=self.channel, pkgname=package_metadata["name"] + channel=self.channel, pkgname=package_name ), - last_update=last_update, + last_update=max(package_dates, default=None), extra_loader_arguments={ - "artifacts": [ - v for k, v in self.packages[package_metadata["name"]].items() - ], + "artifacts": list(self.packages[package_name].values()) }, ) diff --git a/swh/lister/conda/tests/test_lister.py b/swh/lister/conda/tests/test_lister.py --- a/swh/lister/conda/tests/test_lister.py +++ b/swh/lister/conda/tests/test_lister.py @@ -3,36 +3,14 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from swh.lister.conda.lister import CondaLister - - -def test_conda_lister_free_channel(datadir, requests_mock_datadir, swh_scheduler): - lister = CondaLister( - scheduler=swh_scheduler, channel="free", archs=["linux-64", "osx-64", "win-64"] - ) - res = lister.run() - - assert res.pages == 3 - assert res.origins == 11 - - -def test_conda_lister_conda_forge_channel( - datadir, requests_mock_datadir, swh_scheduler -): - lister = CondaLister( - scheduler=swh_scheduler, - url="https://conda.anaconda.org", - channel="conda-forge", - archs=["linux-64"], - ) - res = lister.run() +import pytest - assert res.pages == 1 - assert res.origins == 2 +from swh.lister.conda.lister import CondaLister - scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results - expected_origins = [ +@pytest.fixture +def expected_origins(): + return [ { "url": "https://anaconda.org/conda-forge/21cmfast", "artifacts": [ @@ -75,6 +53,33 @@ }, ] + +def test_conda_lister_free_channel(datadir, requests_mock_datadir, swh_scheduler): + lister = CondaLister( + scheduler=swh_scheduler, channel="free", archs=["linux-64", "osx-64", "win-64"] + ) + res = lister.run() + + assert res.pages == 3 + assert res.origins == 11 + + +def test_conda_lister_conda_forge_channel( + requests_mock_datadir, swh_scheduler, expected_origins +): + lister = CondaLister( + scheduler=swh_scheduler, + url="https://conda.anaconda.org", + channel="conda-forge", + archs=["linux-64"], + ) + res = lister.run() + + assert res.pages == 1 + assert res.origins == 2 + + scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results + assert len(scheduler_origins) == len(expected_origins) assert [ @@ -92,3 +97,23 @@ ) for expected in sorted(expected_origins, key=lambda expected: expected["url"]) ] + + +def test_conda_lister_number_of_yielded_origins( + requests_mock_datadir, swh_scheduler, expected_origins +): + """Check that a single ListedOrigin instance is sent by expected origins.""" + lister = CondaLister( + scheduler=swh_scheduler, + url="https://conda.anaconda.org", + channel="conda-forge", + archs=["linux-64"], + ) + + listed_origins = [] + for page in lister.get_pages(): + listed_origins += list(lister.get_origins_from_page(page)) + + assert sorted([listed_origin.url for listed_origin in listed_origins]) == sorted( + [origin["url"] for origin in expected_origins] + )