diff --git a/swh/loader/package/archive/tests/test_archive.py b/swh/loader/package/archive/tests/test_archive.py --- a/swh/loader/package/archive/tests/test_archive.py +++ b/swh/loader/package/archive/tests/test_archive.py @@ -73,10 +73,10 @@ } _expected_branches_first_visit = { - "HEAD": {"target_type": "alias", "target": "releases/0.1.0",}, - "releases/0.1.0": { + b"HEAD": {"target_type": "alias", "target": b"releases/0.1.0",}, + b"releases/0.1.0": { "target_type": "revision", - "target": "44183488c0774ce3c957fa19ba695cf18a4a42b3", + "target": hash_to_bytes("44183488c0774ce3c957fa19ba695cf18a4a42b3"), }, } diff --git a/swh/loader/package/cran/tests/test_cran.py b/swh/loader/package/cran/tests/test_cran.py --- a/swh/loader/package/cran/tests/test_cran.py +++ b/swh/loader/package/cran/tests/test_cran.py @@ -18,6 +18,7 @@ parse_debian_control, ) from swh.core.tarball import uncompress +from swh.model.hashutil import hash_to_bytes from swh.model.model import TimestampWithTimezone from swh.loader.tests import ( @@ -177,9 +178,9 @@ expected_snapshot = { "id": expected_snapshot_id, "branches": { - "HEAD": {"target": f"releases/{version}", "target_type": "alias"}, - f"releases/{version}": { - "target": "42bdb16facd5140424359c8ce89a28ecfa1ce603", + b"HEAD": {"target": b"releases/2.22-6", "target_type": "alias"}, + b"releases/2.22-6": { + "target": hash_to_bytes("42bdb16facd5140424359c8ce89a28ecfa1ce603"), "target_type": "revision", }, }, @@ -234,9 +235,9 @@ expected_snapshot = { "id": expected_snapshot_id, "branches": { - "HEAD": {"target": f"releases/{version}", "target_type": "alias"}, - f"releases/{version}": { - "target": "42bdb16facd5140424359c8ce89a28ecfa1ce603", + b"HEAD": {"target": b"releases/2.22-6", "target_type": "alias"}, + b"releases/2.22-6": { + "target": hash_to_bytes("42bdb16facd5140424359c8ce89a28ecfa1ce603"), "target_type": "revision", }, }, diff --git a/swh/loader/package/debian/tests/test_debian.py b/swh/loader/package/debian/tests/test_debian.py --- a/swh/loader/package/debian/tests/test_debian.py +++ b/swh/loader/package/debian/tests/test_debian.py @@ -27,6 +27,7 @@ from swh.loader.package.debian.loader import resolve_revision_from +from swh.model.hashutil import hash_to_bytes from swh.model.model import Person @@ -135,9 +136,9 @@ expected_snapshot = { "id": expected_snapshot_id, "branches": { - "releases/stretch/contrib/0.7.2-3": { + b"releases/stretch/contrib/0.7.2-3": { "target_type": "revision", - "target": "2807f5b3f84368b4889a9ae827fe85854ffecf07", + "target": hash_to_bytes("2807f5b3f84368b4889a9ae827fe85854ffecf07"), } }, } # different than the previous loader as no release is done @@ -180,9 +181,9 @@ expected_snapshot = { "id": expected_snapshot_id, "branches": { - "releases/stretch/contrib/0.7.2-3": { + b"releases/stretch/contrib/0.7.2-3": { "target_type": "revision", - "target": "2807f5b3f84368b4889a9ae827fe85854ffecf07", + "target": hash_to_bytes("2807f5b3f84368b4889a9ae827fe85854ffecf07"), } }, } # different than the previous loader as no release is done @@ -382,13 +383,13 @@ expected_snapshot = { "id": expected_snapshot_id, "branches": { - "releases/stretch/contrib/0.7.2-3": { + b"releases/stretch/contrib/0.7.2-3": { "target_type": "revision", - "target": "2807f5b3f84368b4889a9ae827fe85854ffecf07", + "target": hash_to_bytes("2807f5b3f84368b4889a9ae827fe85854ffecf07"), }, - "releases/buster/contrib/0.7.2-4": { + b"releases/buster/contrib/0.7.2-4": { "target_type": "revision", - "target": "8224139c274c984147ef4b09aa0e462c55a10bd3", + "target": hash_to_bytes("8224139c274c984147ef4b09aa0e462c55a10bd3"), }, }, } diff --git a/swh/loader/package/deposit/tests/test_deposit.py b/swh/loader/package/deposit/tests/test_deposit.py --- a/swh/loader/package/deposit/tests/test_deposit.py +++ b/swh/loader/package/deposit/tests/test_deposit.py @@ -170,7 +170,7 @@ revision_id = "637318680351f5d78856d13264faebbd91efe9bb" expected_branches = { - "HEAD": {"target": revision_id, "target_type": "revision",}, + b"HEAD": {"target": hash_to_bytes(revision_id), "target_type": "revision",}, } expected_snapshot = { @@ -245,7 +245,9 @@ assert_last_visit_matches(loader.storage, url, status="full", type="deposit") revision_id = "564d18943d71be80d0d73b43a77cfb205bcde96c" - expected_branches = {"HEAD": {"target": revision_id, "target_type": "revision"}} + expected_branches = { + b"HEAD": {"target": hash_to_bytes(revision_id), "target_type": "revision"} + } expected_snapshot = { "id": expected_snapshot_id, "branches": expected_branches, diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py --- a/swh/loader/package/loader.py +++ b/swh/loader/package/loader.py @@ -144,13 +144,11 @@ ] known_revisions = self.storage.revision_get(revs) - ret = {} - for revision in known_revisions: - if not revision: # revision_get can return None - continue - ret[revision["id"]] = revision["metadata"] - - return ret + return { + revision["id"]: revision["metadata"] + for revision in known_revisions + if revision + } def resolve_revision_from( self, known_artifacts: Dict, artifact_metadata: Dict diff --git a/swh/loader/package/nixguix/tests/test_nixguix.py b/swh/loader/package/nixguix/tests/test_nixguix.py --- a/swh/loader/package/nixguix/tests/test_nixguix.py +++ b/swh/loader/package/nixguix/tests/test_nixguix.py @@ -188,12 +188,12 @@ ) expected_branches = { - "evaluation": { - "target": "cc4e04c26672dd74e5fd0fecb78b435fb55368f7", + b"evaluation": { + "target": hash_to_bytes("cc4e04c26672dd74e5fd0fecb78b435fb55368f7"), "target_type": "revision", }, - "https://github.com/owner-1/repository-1/revision-1.tgz": { - "target": "488ad4e7b8e2511258725063cf43a2b897c503b4", + b"https://github.com/owner-1/repository-1/revision-1.tgz": { + "target": hash_to_bytes("488ad4e7b8e2511258725063cf43a2b897c503b4"), "target_type": "revision", }, } @@ -234,12 +234,12 @@ ) expected_branches = { - "evaluation": { - "target": "cc4e04c26672dd74e5fd0fecb78b435fb55368f7", + b"evaluation": { + "target": hash_to_bytes("cc4e04c26672dd74e5fd0fecb78b435fb55368f7"), "target_type": "revision", }, - "https://github.com/owner-1/repository-1/revision-1.tgz": { - "target": "488ad4e7b8e2511258725063cf43a2b897c503b4", + b"https://github.com/owner-1/repository-1/revision-1.tgz": { + "target": hash_to_bytes("488ad4e7b8e2511258725063cf43a2b897c503b4"), "target_type": "revision", }, } @@ -277,16 +277,16 @@ # second time an url, because of the requests_mock_datadir_visits # fixture, the file has to end with `_visit1`. expected_branches = { - "evaluation": { - "target": "602140776b2ce6c9159bcf52ada73a297c063d5e", + b"evaluation": { + "target": hash_to_bytes("602140776b2ce6c9159bcf52ada73a297c063d5e"), "target_type": "revision", }, - "https://github.com/owner-1/repository-1/revision-1.tgz": { - "target": "488ad4e7b8e2511258725063cf43a2b897c503b4", + b"https://github.com/owner-1/repository-1/revision-1.tgz": { + "target": hash_to_bytes("488ad4e7b8e2511258725063cf43a2b897c503b4"), "target_type": "revision", }, - "https://github.com/owner-2/repository-1/revision-1.tgz": { - "target": "85e0bad74e33e390aaeb74f139853ae3863ee544", + b"https://github.com/owner-2/repository-1/revision-1.tgz": { + "target": hash_to_bytes("85e0bad74e33e390aaeb74f139853ae3863ee544"), "target_type": "revision", }, } @@ -336,12 +336,12 @@ ) expected_branches = { - "https://github.com/owner-1/repository-1/revision-1.tgz": { - "target": "488ad4e7b8e2511258725063cf43a2b897c503b4", + b"https://github.com/owner-1/repository-1/revision-1.tgz": { + "target": hash_to_bytes("488ad4e7b8e2511258725063cf43a2b897c503b4"), "target_type": "revision", }, - "evaluation": { - "target": "cc4e04c26672dd74e5fd0fecb78b435fb55368f7", + b"evaluation": { + "target": hash_to_bytes("cc4e04c26672dd74e5fd0fecb78b435fb55368f7"), "target_type": "revision", }, } @@ -368,8 +368,8 @@ loader.load() expected_branches = { - "evaluation": { - "target": "cc4e04c26672dd74e5fd0fecb78b435fb55368f7", + b"evaluation": { + "target": hash_to_bytes("cc4e04c26672dd74e5fd0fecb78b435fb55368f7"), "target_type": "revision", }, } @@ -416,12 +416,12 @@ } expected_branches = { - "https://github.com/owner-1/repository-1/revision-1.tgz": { - "target": "488ad4e7b8e2511258725063cf43a2b897c503b4", + b"https://github.com/owner-1/repository-1/revision-1.tgz": { + "target": hash_to_bytes("488ad4e7b8e2511258725063cf43a2b897c503b4"), "target_type": "revision", }, - "evaluation": { - "target": "cc4e04c26672dd74e5fd0fecb78b435fb55368f7", + b"evaluation": { + "target": hash_to_bytes("cc4e04c26672dd74e5fd0fecb78b435fb55368f7"), "target_type": "revision", }, } @@ -522,7 +522,7 @@ # mutate snapshot to create a clash snapshot["branches"][artifact_url.encode("utf-8")] = { "target_type": "revision", - "target": old_revision["id"], + "target": hash_to_bytes(old_revision["id"]), } # modify snapshot to actually change revision metadata structure so we simulate diff --git a/swh/loader/package/npm/tests/test_npm.py b/swh/loader/package/npm/tests/test_npm.py --- a/swh/loader/package/npm/tests/test_npm.py +++ b/swh/loader/package/npm/tests/test_npm.py @@ -325,17 +325,17 @@ expected_snapshot = { "id": expected_snapshot_id, "branches": { - "HEAD": {"target": "releases/0.0.4", "target_type": "alias"}, - "releases/0.0.2": { - "target": "d8a1c7474d2956ac598a19f0f27d52f7015f117e", + b"HEAD": {"target": b"releases/0.0.4", "target_type": "alias"}, + b"releases/0.0.2": { + "target": hash_to_bytes("d8a1c7474d2956ac598a19f0f27d52f7015f117e"), "target_type": "revision", }, - "releases/0.0.3": { - "target": "5f9eb78af37ffd12949f235e86fac04898f9f72a", + b"releases/0.0.3": { + "target": hash_to_bytes("5f9eb78af37ffd12949f235e86fac04898f9f72a"), "target_type": "revision", }, - "releases/0.0.4": { - "target": "ba019b192bdb94bd0b5bd68b3a5f92b5acc2239a", + b"releases/0.0.4": { + "target": hash_to_bytes("ba019b192bdb94bd0b5bd68b3a5f92b5acc2239a"), "target_type": "revision", }, }, @@ -426,14 +426,14 @@ expected_snapshot = { "id": "b11ebac8c9d0c9e5063a2df693a18e3aba4b2f92", "branches": { - "HEAD": {"target_type": "alias", "target": "releases/0.1.0"}, - "releases/0.1.0": { + b"HEAD": {"target_type": "alias", "target": b"releases/0.1.0"}, + b"releases/0.1.0": { "target_type": "revision", - "target": "845673bfe8cbd31b1eaf757745a964137e6f9116", + "target": hash_to_bytes("845673bfe8cbd31b1eaf757745a964137e6f9116"), }, - "releases/0.1.1-alpha.14": { + b"releases/0.1.1-alpha.14": { "target_type": "revision", - "target": "05181c12cd8c22035dd31155656826b85745da37", + "target": hash_to_bytes("05181c12cd8c22035dd31155656826b85745da37"), }, }, } @@ -558,10 +558,10 @@ expected_snapshot = { "id": "d6e08e19159f77983242877c373c75222d5ae9dd", "branches": { - "HEAD": {"target_type": "alias", "target": "releases/0.0.1"}, - "releases/0.0.1": { + b"HEAD": {"target_type": "alias", "target": b"releases/0.0.1"}, + b"releases/0.0.1": { "target_type": "revision", - "target": "9e4dd2b40d1b46b70917c0949aa2195c823a648e", + "target": hash_to_bytes("9e4dd2b40d1b46b70917c0949aa2195c823a648e"), }, }, } diff --git a/swh/loader/package/pypi/tests/test_pypi.py b/swh/loader/package/pypi/tests/test_pypi.py --- a/swh/loader/package/pypi/tests/test_pypi.py +++ b/swh/loader/package/pypi/tests/test_pypi.py @@ -373,11 +373,11 @@ assert list(loader.storage.revision_missing(expected_revs)) == [] expected_branches = { - "releases/1.2.0": { - "target": "e445da4da22b31bfebb6ffc4383dbf839a074d21", + b"releases/1.2.0": { + "target": hash_to_bytes("e445da4da22b31bfebb6ffc4383dbf839a074d21"), "target_type": "revision", }, - "HEAD": {"target": "releases/1.2.0", "target_type": "alias",}, + b"HEAD": {"target": b"releases/1.2.0", "target_type": "alias",}, } expected_snapshot = { @@ -454,15 +454,15 @@ assert list(loader.storage.revision_missing(expected_revs)) == [] expected_branches = { - "releases/1.1.0": { - "target": "4c99891f93b81450385777235a37b5e966dd1571", + b"releases/1.1.0": { + "target": hash_to_bytes("4c99891f93b81450385777235a37b5e966dd1571"), "target_type": "revision", }, - "releases/1.2.0": { - "target": "e445da4da22b31bfebb6ffc4383dbf839a074d21", + b"releases/1.2.0": { + "target": hash_to_bytes("e445da4da22b31bfebb6ffc4383dbf839a074d21"), "target_type": "revision", }, - "HEAD": {"target": "releases/1.2.0", "target_type": "alias",}, + b"HEAD": {"target": b"releases/1.2.0", "target_type": "alias",}, } expected_snapshot = { @@ -504,15 +504,15 @@ } == stats expected_branches = { - "releases/1.1.0": { - "target": "4c99891f93b81450385777235a37b5e966dd1571", + b"releases/1.1.0": { + "target": hash_to_bytes("4c99891f93b81450385777235a37b5e966dd1571"), "target_type": "revision", }, - "releases/1.2.0": { - "target": "e445da4da22b31bfebb6ffc4383dbf839a074d21", + b"releases/1.2.0": { + "target": hash_to_bytes("e445da4da22b31bfebb6ffc4383dbf839a074d21"), "target_type": "revision", }, - "HEAD": {"target": "releases/1.2.0", "target_type": "alias",}, + b"HEAD": {"target": b"releases/1.2.0", "target_type": "alias",}, } expected_snapshot = { @@ -642,19 +642,19 @@ assert list(loader.storage.revision_missing(expected_revs)) == [] expected_branches = { - "releases/1.1.0": { - "target": "4c99891f93b81450385777235a37b5e966dd1571", + b"releases/1.1.0": { + "target": hash_to_bytes("4c99891f93b81450385777235a37b5e966dd1571"), "target_type": "revision", }, - "releases/1.2.0": { - "target": "e445da4da22b31bfebb6ffc4383dbf839a074d21", + b"releases/1.2.0": { + "target": hash_to_bytes("e445da4da22b31bfebb6ffc4383dbf839a074d21"), "target_type": "revision", }, - "releases/1.3.0": { - "target": "51247143b01445c9348afa9edfae31bf7c5d86b1", + b"releases/1.3.0": { + "target": hash_to_bytes("51247143b01445c9348afa9edfae31bf7c5d86b1"), "target_type": "revision", }, - "HEAD": {"target": "releases/1.3.0", "target_type": "alias",}, + b"HEAD": {"target": b"releases/1.3.0", "target_type": "alias",}, } expected_snapshot = { "id": expected_snapshot_id2, @@ -700,12 +700,12 @@ } expected_branches = { - "releases/1.1.0/nexter-1.1.0.zip": { - "target": "4c99891f93b81450385777235a37b5e966dd1571", + b"releases/1.1.0/nexter-1.1.0.zip": { + "target": hash_to_bytes("4c99891f93b81450385777235a37b5e966dd1571"), "target_type": "revision", }, - "releases/1.1.0/nexter-1.1.0.tar.gz": { - "target": "0bf88f5760cca7665d0af4d6575d9301134fe11a", + b"releases/1.1.0/nexter-1.1.0.tar.gz": { + "target": hash_to_bytes("0bf88f5760cca7665d0af4d6575d9301134fe11a"), "target_type": "revision", }, } diff --git a/swh/loader/tests/test_init.py b/swh/loader/tests/test_init.py --- a/swh/loader/tests/test_init.py +++ b/swh/loader/tests/test_init.py @@ -386,7 +386,10 @@ unexpected_snapshot = { "id": "2498dbf535f882bc7f9a18fb16c9ad27fda7bab7", # id is correct "branches": { - "master": {"target": hash_hex, "target_type": "release",} # wrong branch + b"master": { + "target": hash_to_bytes(hash_hex), # wrong branch + "target_type": "release", + } }, }