diff --git a/swh/lister/puppet/__init__.py b/swh/lister/puppet/__init__.py --- a/swh/lister/puppet/__init__.py +++ b/swh/lister/puppet/__init__.py @@ -38,7 +38,7 @@ "https://forge.puppet.com/modules/{owner}/{pkgname}" -For each origin `last_update`is set via the module "updated_at" value. +For each origin `last_update` is set via the module "updated_at" value. As the api also returns all existing versions for a package, we build an `artifacts` dict in `extra_loader_arguments` with the archive tarball corresponding to each existing versions. @@ -47,20 +47,27 @@ https://forge.puppet.com/modules/electrical/file_concat:: { - "artifacts": { - "1.0.0": { - "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", # noqa: B950 - "version": "1.0.0", - "filename": "electrical-file_concat-1.0.0.tar.gz", - "last_update": "2015-04-09T12:03:13-07:00", - }, - "1.0.1": { + "artifacts": [ + { "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz", # noqa: B950 "version": "1.0.1", "filename": "electrical-file_concat-1.0.1.tar.gz", "last_update": "2015-04-17T01:03:46-07:00", + "checksums": { + "md5": "74901a89544134478c2dfde5efbb7f14", + "sha256": "15e973613ea038d8a4f60bafe2d678f88f53f3624c02df3157c0043f4a400de6", # noqa: B950 + }, + }, + { + "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", # noqa: B950 + "version": "1.0.0", + "filename": "electrical-file_concat-1.0.0.tar.gz", + "last_update": "2015-04-09T12:03:13-07:00", + "checksums": { + "length": 13289, + }, }, - } + ], } Running tests diff --git a/swh/lister/puppet/lister.py b/swh/lister/puppet/lister.py --- a/swh/lister/puppet/lister.py +++ b/swh/lister/puppet/lister.py @@ -76,7 +76,7 @@ pkgname = entry["name"] owner = entry["owner"]["slug"] url = f"https://forge.puppet.com/modules/{owner}/{pkgname}" - artifacts = {} + artifacts = [] for release in entry["releases"]: # Build an artifact entry following original-artifacts-json specification # https://docs.softwareheritage.org/devel/swh-storage/extrinsic-metadata-specification.html#original-artifacts-json # noqa: B950 @@ -92,15 +92,17 @@ # use file length as basic content check instead checksums["length"] = release["file_size"] - artifacts[release["version"]] = { - "filename": release["file_uri"].split("/")[-1], - "url": urljoin(self.BASE_URL, release["file_uri"]), - "version": release["version"], - "last_update": datetime.strptime( - release["created_at"], dt_parse_pattern - ).isoformat(), - "checksums": checksums, - } + artifacts.append( + { + "filename": release["file_uri"].split("/")[-1], + "url": urljoin(self.BASE_URL, release["file_uri"]), + "version": release["version"], + "last_update": datetime.strptime( + release["created_at"], dt_parse_pattern + ).isoformat(), + "checksums": checksums, + } + ) yield ListedOrigin( lister_id=self.lister_obj.id, diff --git a/swh/lister/puppet/tests/test_lister.py b/swh/lister/puppet/tests/test_lister.py --- a/swh/lister/puppet/tests/test_lister.py +++ b/swh/lister/puppet/tests/test_lister.py @@ -7,19 +7,11 @@ # flake8: noqa: B950 -expected_origins = { - "https://forge.puppet.com/modules/electrical/file_concat": { - "artifacts": { - "1.0.0": { - "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", - "version": "1.0.0", - "filename": "electrical-file_concat-1.0.0.tar.gz", - "last_update": "2015-04-09T12:03:13-07:00", - "checksums": { - "length": 13289, - }, - }, - "1.0.1": { +expected_origins = [ + { + "url": "https://forge.puppet.com/modules/electrical/file_concat", + "artifacts": [ + { "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz", "version": "1.0.1", "filename": "electrical-file_concat-1.0.1.tar.gz", @@ -29,29 +21,21 @@ "sha256": "15e973613ea038d8a4f60bafe2d678f88f53f3624c02df3157c0043f4a400de6", }, }, - } - }, - "https://forge.puppet.com/modules/puppetlabs/puppetdb": { - "artifacts": { - "1.0.0": { - "url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-1.0.0.tar.gz", + { + "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", "version": "1.0.0", - "filename": "puppetlabs-puppetdb-1.0.0.tar.gz", - "last_update": "2012-09-19T16:51:22-07:00", - "checksums": { - "length": 16336, - }, - }, - "7.9.0": { - "url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.9.0.tar.gz", - "version": "7.9.0", - "filename": "puppetlabs-puppetdb-7.9.0.tar.gz", - "last_update": "2021-06-24T07:48:54-07:00", + "filename": "electrical-file_concat-1.0.0.tar.gz", + "last_update": "2015-04-09T12:03:13-07:00", "checksums": { - "length": 42773, + "length": 13289, }, }, - "7.10.0": { + ], + }, + { + "url": "https://forge.puppet.com/modules/puppetlabs/puppetdb", + "artifacts": [ + { "url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.10.0.tar.gz", "version": "7.10.0", "filename": "puppetlabs-puppetdb-7.10.0.tar.gz", @@ -61,20 +45,30 @@ "sha256": "49b1a542fbd2a1378c16cb04809e0f88bf4f3e45979532294fb1f03f56c97fbb", }, }, - } - }, - "https://forge.puppet.com/modules/saz/memcached": { - "artifacts": { - "1.0.0": { - "url": "https://forgeapi.puppet.com/v3/files/saz-memcached-1.0.0.tar.gz", + { + "url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.9.0.tar.gz", + "version": "7.9.0", + "filename": "puppetlabs-puppetdb-7.9.0.tar.gz", + "last_update": "2021-06-24T07:48:54-07:00", + "checksums": { + "length": 42773, + }, + }, + { + "url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-1.0.0.tar.gz", "version": "1.0.0", - "filename": "saz-memcached-1.0.0.tar.gz", - "last_update": "2011-11-20T13:40:30-08:00", + "filename": "puppetlabs-puppetdb-1.0.0.tar.gz", + "last_update": "2012-09-19T16:51:22-07:00", "checksums": { - "length": 2472, + "length": 16336, }, }, - "8.1.0": { + ], + }, + { + "url": "https://forge.puppet.com/modules/saz/memcached", + "artifacts": [ + { "url": "https://forgeapi.puppet.com/v3/files/saz-memcached-8.1.0.tar.gz", "version": "8.1.0", "filename": "saz-memcached-8.1.0.tar.gz", @@ -84,9 +78,18 @@ "sha256": "883d6186e91c2c3fed13ae2009c3aa596657f6707b76f1f7efc6203c6e4ae986", }, }, - } + { + "url": "https://forgeapi.puppet.com/v3/files/saz-memcached-1.0.0.tar.gz", + "version": "1.0.0", + "filename": "saz-memcached-1.0.0.tar.gz", + "last_update": "2011-11-20T13:40:30-08:00", + "checksums": { + "length": 2472, + }, + }, + ], }, -} +] def test_puppet_lister(datadir, requests_mock_datadir, swh_scheduler): @@ -100,7 +103,18 @@ assert len(scheduler_origins) == len(expected_origins) - for origin in scheduler_origins: - assert origin.visit_type == "puppet" - assert origin.url in expected_origins - assert origin.extra_loader_arguments == expected_origins[origin.url] + assert [ + ( + scheduled.visit_type, + scheduled.url, + scheduled.extra_loader_arguments["artifacts"], + ) + for scheduled in sorted(scheduler_origins, key=lambda scheduled: scheduled.url) + ] == [ + ( + "puppet", + expected["url"], + expected["artifacts"], + ) + for expected in sorted(expected_origins, key=lambda expected: expected["url"]) + ]