diff --git a/swh/loader/package/nixguix/loader.py b/swh/loader/package/nixguix/loader.py --- a/swh/loader/package/nixguix/loader.py +++ b/swh/loader/package/nixguix/loader.py @@ -41,6 +41,10 @@ s = retrieve_sources(url) self.sources = s['sources'] self.provider_url = url + + self._integrityByUrl = {s['url'][0]: s['integrity'] + for s in self.sources} + # The revision used to create the sources.json file. For Nix, # this revision belongs to the github.com/nixos/nixpkgs # repository @@ -49,24 +53,32 @@ # Note: this could be renamed get_artifacts in the PackageLoader # base class. def get_versions(self): + """The first mirror of the mirror list is used as branch name in the + snapshot. + + """ + return self._integrityByUrl.keys() + + # Note: this could be renamed get_artifact_info in the PackageLoader + # base class. + def get_package_info(self, url): # TODO: try all mirrors and not only the first one. A source # can be fetched from several urls, called mirrors. We # currently only use the first one, but if the first one # fails, we should try the second one and so on. - return [s['url'][0] for s in self.sources] - - # Note: this could be renamed get_artifact_info in the PackageLoader - # base class. - def get_package_info(self, source): - # TODO: we need to provide the sha256 of the source also - yield source, {'url': source, 'raw': {'url': source}} + integrity = self._integrityByUrl[url] + yield url, {'url': url, + 'raw': { + 'url': url, + 'integrity': integrity}} def resolve_revision_from( self, known_artifacts: Dict, artifact_metadata: Dict) \ -> Optional[bytes]: + for rev_id, known_artifact in known_artifacts.items(): - known_url = known_artifact['extrinsic']['raw']['url'] - if artifact_metadata['url'] == known_url: + known_integrity = known_artifact['extrinsic']['raw']['integrity'] + if artifact_metadata['integrity'] == known_integrity: return rev_id return None diff --git a/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json --- a/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json +++ b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json @@ -2,7 +2,8 @@ "sources": [ { "type": "url", - "url": [ "https://fail.com/truncated-archive.tgz" ] + "url": [ "https://fail.com/truncated-archive.tgz" ], + "integrity": "sha256-UB+RzIn63O0WxzqohYeWZRRzYCxyK7Kfhqi6WI0P8bE=" } ], "version": 1, diff --git a/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json --- a/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json +++ b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json @@ -2,11 +2,13 @@ "sources": [ { "type": "url", - "url": [ "https://github.com/owner-1/repository-1/revision-1.tgz" ] + "url": [ "https://github.com/owner-1/repository-1/revision-1.tgz" ], + "integrity": "sha256-3vm2Nt+O4zHf3Ovd/qsv1gKTEUwodX9FLxlrQdry0zs=" }, { "type": "url", - "url": [ "https://example.com/file.txt" ] + "url": [ "https://example.com/file.txt" ], + "integrity": "sha256-Q0copBCnj1b8G1iZw1k0NuYasMcx6QctleltspAgXlM=" } ], "version": 1, diff --git a/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1 b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1 --- a/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1 +++ b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1 @@ -2,15 +2,18 @@ "sources": [ { "type": "url", - "url": [ "https://github.com/owner-1/repository-1/revision-1.tgz" ] + "url": [ "https://github.com/owner-1/repository-1/revision-1.tgz" ], + "integrity": "sha256-3vm2Nt+O4zHf3Ovd/qsv1gKTEUwodX9FLxlrQdry0zs=" }, { "type": "url", - "url": [ "https://github.com/owner-2/repository-1/revision-1.tgz" ] + "url": [ "https://github.com/owner-2/repository-1/revision-1.tgz" ], + "integrity": "sha256-+vRlzTcnhMlynJGGMuAgMnUGdjpSqGabhcQ/SlRplAE=" }, { "type": "url", - "url": [ "https://example.com/file.txt" ] + "url": [ "https://example.com/file.txt" ], + "integrity": "sha256-Q0copBCnj1b8G1iZw1k0NuYasMcx6QctleltspAgXlM=" } ], "version": 1, diff --git a/swh/loader/package/nixguix/tests/test_functional.py b/swh/loader/package/nixguix/tests/test_functional.py --- a/swh/loader/package/nixguix/tests/test_functional.py +++ b/swh/loader/package/nixguix/tests/test_functional.py @@ -88,7 +88,6 @@ loader = NixGuixLoader(sources_url) load_status = loader.load() - loader = NixGuixLoader(sources_url) loader.load() expected_snapshot_id = '0c5881c74283793ebe9a09a105a9381e41380383' assert load_status == { @@ -218,13 +217,17 @@ loader = NixGuixLoader(sources_url) known_artifacts = { - 'id1': {'extrinsic': {'raw': {'url': "url1"}}}, - 'id2': {'extrinsic': {'raw': {'url': "url2"}}} + 'id1': {'extrinsic': {'raw': { + 'url': "url1", + 'integrity': 'integrity1'}}}, + 'id2': {'extrinsic': {'raw': { + 'url': "url2", + 'integrity': 'integrity2'}}}, } - metadata = {'url': 'url1'} + metadata = {'url': 'url1', 'integrity': 'integrity1'} assert loader.resolve_revision_from(known_artifacts, metadata) == 'id1' - metadata = {'url': 'url3'} + metadata = {'url': 'url3', 'integrity': 'integrity3'} assert loader.resolve_revision_from(known_artifacts, metadata) == None # noqa