diff --git a/swh/loader/package/nixguix/loader.py b/swh/loader/package/nixguix/loader.py --- a/swh/loader/package/nixguix/loader.py +++ b/swh/loader/package/nixguix/loader.py @@ -26,7 +26,14 @@ raise ValueError("Got %d HTTP code on %s", response.status_code, url) - return json.loads(response.content.decode('utf-8')) + sources = json.loads(response.content.decode('utf-8')) + + byIntegrity = {} + for s in sources['sources']: + byIntegrity[s['integrity']] = s['url'] + sources['byIntegrity'] = byIntegrity + + return sources class NixGuixLoader(PackageLoader): @@ -39,7 +46,7 @@ def __init__(self, url): super().__init__(url=url) s = retrieve_sources(url) - self.sources = s['sources'] + self.sources = s['byIntegrity'] self.provider_url = url # The revision used to create the sources.json file. For Nix, # this revision belongs to the github.com/nixos/nixpkgs @@ -49,24 +56,27 @@ # Note: this could be renamed get_artifacts in the PackageLoader # base class. def get_versions(self): + return self.sources.keys() + + # Note: this could be renamed get_artifact_info in the PackageLoader + # base class. + def get_package_info(self, integrity): # TODO: try all mirrors and not only the first one. A source # can be fetched from several urls, called mirrors. We # currently only use the first one, but if the first one # fails, we should try the second one and so on. - return [s['url'][0] for s in self.sources] - - # Note: this could be renamed get_artifact_info in the PackageLoader - # base class. - def get_package_info(self, source): - # TODO: we need to provide the sha256 of the source also - yield source, {'url': source, 'raw': {'url': source}} + url = self.sources[integrity][0] + yield integrity, {'url': url, + 'raw': { + 'url': url, + 'integrity': integrity}} def resolve_revision_from( self, known_artifacts: Dict, artifact_metadata: Dict) \ -> Optional[bytes]: for rev_id, known_artifact in known_artifacts.items(): - known_url = known_artifact['extrinsic']['raw']['url'] - if artifact_metadata['url'] == known_url: + known_integrity = known_artifact['extrinsic']['raw']['integrity'] + if artifact_metadata['integrity'] == known_integrity: return rev_id return None diff --git a/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json --- a/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json +++ b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json @@ -2,7 +2,8 @@ "sources": [ { "type": "url", - "url": [ "https://fail.com/truncated-archive.tgz" ] + "url": [ "https://fail.com/truncated-archive.tgz" ], + "integrity": "sha256-UB+RzIn63O0WxzqohYeWZRRzYCxyK7Kfhqi6WI0P8bE=" } ], "version": 1, diff --git a/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json --- a/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json +++ b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json @@ -2,11 +2,13 @@ "sources": [ { "type": "url", - "url": [ "https://github.com/owner-1/repository-1/revision-1.tgz" ] + "url": [ "https://github.com/owner-1/repository-1/revision-1.tgz" ], + "integrity": "sha256-3vm2Nt+O4zHf3Ovd/qsv1gKTEUwodX9FLxlrQdry0zs=" }, { "type": "url", - "url": [ "https://example.com/file.txt" ] + "url": [ "https://example.com/file.txt" ], + "integrity": "sha256-Q0copBCnj1b8G1iZw1k0NuYasMcx6QctleltspAgXlM=" } ], "version": 1, diff --git a/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1 b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1 --- a/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1 +++ b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1 @@ -2,15 +2,18 @@ "sources": [ { "type": "url", - "url": [ "https://github.com/owner-1/repository-1/revision-1.tgz" ] + "url": [ "https://github.com/owner-1/repository-1/revision-1.tgz" ], + "integrity": "sha256-3vm2Nt+O4zHf3Ovd/qsv1gKTEUwodX9FLxlrQdry0zs=" }, { "type": "url", - "url": [ "https://github.com/owner-2/repository-1/revision-1.tgz" ] + "url": [ "https://github.com/owner-2/repository-1/revision-1.tgz" ], + "integrity": "sha256-+vRlzTcnhMlynJGGMuAgMnUGdjpSqGabhcQ/SlRplAE=" }, { "type": "url", - "url": [ "https://example.com/file.txt" ] + "url": [ "https://example.com/file.txt" ], + "integrity": "sha256-Q0copBCnj1b8G1iZw1k0NuYasMcx6QctleltspAgXlM=" } ], "version": 1, diff --git a/swh/loader/package/nixguix/tests/test_functional.py b/swh/loader/package/nixguix/tests/test_functional.py --- a/swh/loader/package/nixguix/tests/test_functional.py +++ b/swh/loader/package/nixguix/tests/test_functional.py @@ -70,7 +70,7 @@ loader = NixGuixLoader(sources_url) loader_status = loader.load() - urls = [s['url'][0] for s in loader.sources] + urls = [urls[0] for _, urls in loader.sources.items()] assert "https://example.com/file.txt" in urls assert loader_status['status'] == 'eventful' @@ -90,7 +90,7 @@ loader = NixGuixLoader(sources_url) loader.load() - expected_snapshot_id = '0c5881c74283793ebe9a09a105a9381e41380383' + expected_snapshot_id = 'f12e63afc45221914f4d7122fc90dbaf68de323e' assert load_status == { 'status': 'eventful', 'snapshot_id': expected_snapshot_id @@ -100,7 +100,7 @@ 'target': 'cc4e04c26672dd74e5fd0fecb78b435fb55368f7', 'target_type': 'revision' }, - 'https://github.com/owner-1/repository-1/revision-1.tgz': { + 'sha256-3vm2Nt+O4zHf3Ovd/qsv1gKTEUwodX9FLxlrQdry0zs=': { 'target': '488ad4e7b8e2511258725063cf43a2b897c503b4', 'target_type': 'revision' }, @@ -132,7 +132,7 @@ """ loader = NixGuixLoader(sources_url) load_status = loader.load() - expected_snapshot_id = '0c5881c74283793ebe9a09a105a9381e41380383' + expected_snapshot_id = 'f12e63afc45221914f4d7122fc90dbaf68de323e' assert load_status == { 'status': 'eventful', 'snapshot_id': expected_snapshot_id @@ -143,7 +143,7 @@ 'target': 'cc4e04c26672dd74e5fd0fecb78b435fb55368f7', 'target_type': 'revision' }, - 'https://github.com/owner-1/repository-1/revision-1.tgz': { + 'sha256-3vm2Nt+O4zHf3Ovd/qsv1gKTEUwodX9FLxlrQdry0zs=': { 'target': '488ad4e7b8e2511258725063cf43a2b897c503b4', 'target_type': 'revision' } @@ -170,7 +170,7 @@ loader = NixGuixLoader(sources_url) load_status = loader.load() - expected_snapshot_id = 'b0bfa75cbd0cc90aac3b9e95fb0f59c731176d97' + expected_snapshot_id = 'db04ce695f0e436a817a9c27ada343a2b475b511' assert load_status == { 'status': 'eventful', 'snapshot_id': expected_snapshot_id @@ -184,11 +184,11 @@ 'target': '602140776b2ce6c9159bcf52ada73a297c063d5e', 'target_type': 'revision' }, - 'https://github.com/owner-1/repository-1/revision-1.tgz': { + 'sha256-3vm2Nt+O4zHf3Ovd/qsv1gKTEUwodX9FLxlrQdry0zs=': { 'target': '488ad4e7b8e2511258725063cf43a2b897c503b4', 'target_type': 'revision' }, - 'https://github.com/owner-2/repository-1/revision-1.tgz': { + 'sha256-+vRlzTcnhMlynJGGMuAgMnUGdjpSqGabhcQ/SlRplAE=': { 'target': '85e0bad74e33e390aaeb74f139853ae3863ee544', 'target_type': 'revision' } @@ -218,13 +218,17 @@ loader = NixGuixLoader(sources_url) known_artifacts = { - 'id1': {'extrinsic': {'raw': {'url': "url1"}}}, - 'id2': {'extrinsic': {'raw': {'url': "url2"}}} + 'id1': {'extrinsic': {'raw': { + 'url': "url1", + 'integrity': 'integrity1'}}}, + 'id2': {'extrinsic': {'raw': { + 'url': "url2", + 'integrity': 'integrity2'}}}, } - metadata = {'url': 'url1'} + metadata = {'url': 'url1', 'integrity': 'integrity1'} assert loader.resolve_revision_from(known_artifacts, metadata) == 'id1' - metadata = {'url': 'url3'} + metadata = {'url': 'url3', 'integrity': 'integrity3'} assert loader.resolve_revision_from(known_artifacts, metadata) == None # noqa @@ -234,7 +238,7 @@ assert res['status'] == 'eventful' expected_branches = { - 'https://github.com/owner-1/repository-1/revision-1.tgz': { + 'sha256-3vm2Nt+O4zHf3Ovd/qsv1gKTEUwodX9FLxlrQdry0zs=': { 'target': '488ad4e7b8e2511258725063cf43a2b897c503b4', 'target_type': 'revision', }, @@ -245,7 +249,7 @@ } expected_snapshot = { - 'id': '0c5881c74283793ebe9a09a105a9381e41380383', + 'id': 'f12e63afc45221914f4d7122fc90dbaf68de323e', 'branches': expected_branches, } diff --git a/swh/loader/package/nixguix/tests/test_tasks.py b/swh/loader/package/nixguix/tests/test_tasks.py --- a/swh/loader/package/nixguix/tests/test_tasks.py +++ b/swh/loader/package/nixguix/tests/test_tasks.py @@ -13,6 +13,7 @@ 'swh.loader.package.nixguix.loader.retrieve_sources') mock_retrieve_sources.return_value = { 'sources': [], + 'byIntegrity': {}, 'revision': 'some-revision' }