diff --git a/swh/loader/tests/__init__.py b/swh/loader/tests/__init__.py --- a/swh/loader/tests/__init__.py +++ b/swh/loader/tests/__init__.py @@ -205,6 +205,7 @@ f"Branch/Release(s) {missing_objs} should exist in storage" ) + # first level dirs exist? dirs = objects_by_target_type.get(TargetType.DIRECTORY) if dirs: not_found = list(storage.directory_missing(dirs)) @@ -216,6 +217,39 @@ f"Missing directories {missing_objs}: " "(revision exists, directory target does not)" ) + for dir_ in dirs: # retrieve new objects to check for existence + paths = storage.directory_ls(dir_, recursive=True) + for path in paths: + if path["type"] == "dir": + target_type = TargetType.DIRECTORY + else: + target_type = TargetType.CONTENT + target = path["target"] + objects_by_target_type[target_type].append(target) + object_to_branch[target] = dir_ + + # check nested directories + dirs = objects_by_target_type.get(TargetType.DIRECTORY) + if dirs: + not_found = list(storage.directory_missing(dirs)) + if not_found: + missing_objs = ", ".join( + str((object_to_branch[dir_].hex(), dir_.hex())) for dir_ in not_found + ) + raise InexistentObjectsError( + f"Missing directories {missing_objs}: " + "(revision exists, directory target does not)" + ) + + # check contents directories + cnts = objects_by_target_type.get(TargetType.CONTENT) + if cnts: + not_found = list(storage.content_missing_per_sha1_git(cnts)) + if not_found: + missing_objs = ", ".join( + str((object_to_branch[cnt].hex(), cnt.hex())) for cnt in not_found + ) + raise InexistentObjectsError(f"Missing contents {missing_objs}") # for retro compat, returned the dict, remove when clients are migrated return snapshot_dict diff --git a/swh/loader/tests/test_init.py b/swh/loader/tests/test_init.py --- a/swh/loader/tests/test_init.py +++ b/swh/loader/tests/test_init.py @@ -363,11 +363,9 @@ 4. snapshot is found in storage, targeted revision does not exist 5. snapshot is found in storage, targeted revision exists but the directory the revision targets does not exist - 6. snapshot is found in storage, targeted release does not exist - - The following are not dealt with yet: - 7. snapshot is found in storage, nested targeted directories does not exist - 8. snapshot is found in storage, nested targeted contents does not exist + 6. snapshot is found in storage, target revision exists, targeted directory by the + revision exist. Content targeted by the directory does not exist. + 7. snapshot is found in storage, targeted release does not exist """ snap_id_hex = "2498dbf535f882bc7f9a18fb16c9ad27fda7bab7" @@ -473,9 +471,35 @@ assert DIRECTORY.id == REVISION.directory swh_storage.directory_add([DIRECTORY]) - # 6. snapshot is found in storage, targeted release does not exist + # 6. snapshot is found in storage, target revision exists, targeted directory by the + # revision exist. Content targeted by the directory does not exist. + + assert DIRECTORY.entries[0].target == CONTENT.sha1_git + not_found = list(swh_storage.content_missing_per_sha1_git([CONTENT.sha1_git])) + assert len(not_found) == 1 + + swh_storage.directory_add([DIRECTORY.to_dict()]) snapshot3 = Snapshot( + id=hash_to_bytes("091456f535f882bc7f9a18fb16c9ad27fda7bab7"), + branches={ + b"alias": SnapshotBranch(target=b"HEAD", target_type=TargetType.ALIAS,), + b"HEAD": SnapshotBranch( + target=REVISION.id, target_type=TargetType.REVISION, + ), + }, + ) + + swh_storage.snapshot_add([snapshot3.to_dict()]) + with pytest.raises(InexistentObjectsError, match="Missing content(s)"): + check_snapshot(snapshot3, swh_storage) + + # 7. snapshot is found in storage, targeted release does not exist + + # release targets the revisions which exists + assert RELEASE.target == REVISION.id + + snapshot4 = Snapshot( id=hash_to_bytes("789666f535f882bc7f9a18fb16c9ad27fda7bab7"), branches={ b"alias": SnapshotBranch(target=b"HEAD", target_type=TargetType.ALIAS,), @@ -488,7 +512,7 @@ }, ) - swh_storage.snapshot_add([snapshot3]) + swh_storage.snapshot_add([snapshot4]) with pytest.raises(InexistentObjectsError, match="Branch/Release"): - check_snapshot(snapshot3, swh_storage) + check_snapshot(snapshot4, swh_storage)