Changeset View
Changeset View
Standalone View
Standalone View
swh/vault/tests/test_cookers.py
Show All 19 Lines | |||||
import dulwich.fastexport | import dulwich.fastexport | ||||
import dulwich.index | import dulwich.index | ||||
import dulwich.objects | import dulwich.objects | ||||
import dulwich.porcelain | import dulwich.porcelain | ||||
import dulwich.repo | import dulwich.repo | ||||
import pytest | import pytest | ||||
from swh.loader.git.from_disk import GitLoaderFromDisk | from swh.loader.git.from_disk import GitLoaderFromDisk | ||||
from swh.model import from_disk, hashutil | from swh.model import from_disk, hashutil, identifiers | ||||
from swh.model.model import ( | from swh.model.model import ( | ||||
Directory, | Directory, | ||||
DirectoryEntry, | DirectoryEntry, | ||||
Person, | Person, | ||||
Revision, | Revision, | ||||
RevisionType, | RevisionType, | ||||
TimestampWithTimezone, | TimestampWithTimezone, | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines | def commit(self, message="Commit test\n", ref=b"HEAD"): | ||||
# committing on another branch leaves | # committing on another branch leaves | ||||
# dangling files in index | # dangling files in index | ||||
if ref != b"HEAD": | if ref != b"HEAD": | ||||
# XXX this should work (but does not) | # XXX this should work (but does not) | ||||
# dulwich.porcelain.reset(self.repo, 'hard') | # dulwich.porcelain.reset(self.repo, 'hard') | ||||
self.git_shell("reset", "--hard", "HEAD") | self.git_shell("reset", "--hard", "HEAD") | ||||
return ret | return ret | ||||
def tag(self, name, target=b"HEAD", message=None): | |||||
dulwich.porcelain.tag_create( | |||||
self.repo, | |||||
name, | |||||
message=message, | |||||
annotated=message is not None, | |||||
objectish=target, | |||||
) | |||||
def merge(self, parent_sha_list, message="Merge branches."): | def merge(self, parent_sha_list, message="Merge branches."): | ||||
self.git_shell( | self.git_shell( | ||||
"merge", | "merge", | ||||
"--allow-unrelated-histories", | "--allow-unrelated-histories", | ||||
"-m", | "-m", | ||||
message, | message, | ||||
*[p.decode() for p in parent_sha_list], | *[p.decode() for p in parent_sha_list], | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 151 Lines • ▼ Show 20 Lines | def cook_extract_revision_gitfast(storage, obj_id, fsck=True): | ||||
test_repo = TestRepo() | test_repo = TestRepo() | ||||
with cook_stream_revision_gitfast(storage, obj_id) as stream, test_repo as p: | with cook_stream_revision_gitfast(storage, obj_id) as stream, test_repo as p: | ||||
processor = dulwich.fastexport.GitImportProcessor(test_repo.repo) | processor = dulwich.fastexport.GitImportProcessor(test_repo.repo) | ||||
processor.import_stream(stream) | processor.import_stream(stream) | ||||
yield test_repo, p | yield test_repo, p | ||||
@contextlib.contextmanager | @contextlib.contextmanager | ||||
def cook_extract_revision_git_bare(storage, obj_id, fsck=True): | def cook_extract_git_bare(storage, swhid, fsck=True): | ||||
"""Context manager that cooks a revision and extract it, | """Context manager that cooks a revision and extract it, | ||||
using GitBareCooker""" | using GitBareCooker""" | ||||
backend = unittest.mock.MagicMock() | backend = unittest.mock.MagicMock() | ||||
backend.storage = storage | backend.storage = storage | ||||
# Cook the object | # Cook the object | ||||
cooker = GitBareCooker("revision", obj_id, backend=backend, storage=storage) | cooker = GitBareCooker( | ||||
swhid.object_type.name.lower(), | |||||
swhid.object_id, | |||||
backend=backend, | |||||
storage=storage, | |||||
) | |||||
cooker.use_fsck = fsck # Some tests try edge-cases that git-fsck rejects | cooker.use_fsck = fsck # Some tests try edge-cases that git-fsck rejects | ||||
cooker.fileobj = io.BytesIO() | cooker.fileobj = io.BytesIO() | ||||
assert cooker.check_exists() | assert cooker.check_exists() | ||||
cooker.prepare_bundle() | cooker.prepare_bundle() | ||||
cooker.fileobj.seek(0) | cooker.fileobj.seek(0) | ||||
# Extract it | # Extract it | ||||
with tempfile.TemporaryDirectory(prefix="tmp-vault-extract-") as td: | with tempfile.TemporaryDirectory(prefix="tmp-vault-extract-") as td: | ||||
with tarfile.open(fileobj=cooker.fileobj, mode="r") as tar: | with tarfile.open(fileobj=cooker.fileobj, mode="r") as tar: | ||||
tar.extractall(td) | tar.extractall(td) | ||||
# Clone it with Dulwich | # Clone it with Dulwich | ||||
with tempfile.TemporaryDirectory(prefix="tmp-vault-clone-") as clone_dir: | with tempfile.TemporaryDirectory(prefix="tmp-vault-clone-") as clone_dir: | ||||
clone_dir = pathlib.Path(clone_dir) | clone_dir = pathlib.Path(clone_dir) | ||||
subprocess.check_call( | subprocess.check_call( | ||||
[ | ["git", "clone", os.path.join(td, f"{swhid}.git"), clone_dir,] | ||||
"git", | |||||
"clone", | |||||
os.path.join(td, f"swh:1:rev:{obj_id.hex()}.git"), | |||||
clone_dir, | |||||
] | |||||
) | ) | ||||
test_repo = TestRepo(clone_dir) | test_repo = TestRepo(clone_dir) | ||||
with test_repo: | with test_repo: | ||||
yield test_repo, clone_dir | yield test_repo, clone_dir | ||||
@contextlib.contextmanager | |||||
def cook_extract_revision_git_bare(storage, obj_id, fsck=True): | |||||
with cook_extract_git_bare( | |||||
storage, | |||||
identifiers.CoreSWHID( | |||||
object_type=identifiers.ObjectType.REVISION, object_id=obj_id | |||||
), | |||||
fsck=fsck, | |||||
) as res: | |||||
yield res | |||||
@pytest.fixture( | @pytest.fixture( | ||||
scope="module", | scope="module", | ||||
params=[cook_extract_revision_gitfast, cook_extract_revision_git_bare], | params=[cook_extract_revision_gitfast, cook_extract_revision_git_bare], | ||||
) | ) | ||||
def cook_extract_revision(request): | def cook_extract_revision(request): | ||||
"""A fixture that is instantiated as either cook_extract_revision_gitfast or | """A fixture that is instantiated as either cook_extract_revision_gitfast or | ||||
cook_extract_revision_git_bare.""" | cook_extract_revision_git_bare.""" | ||||
return request.param | return request.param | ||||
@contextlib.contextmanager | |||||
def cook_extract_snapshot_git_bare(storage, obj_id, fsck=True): | |||||
with cook_extract_git_bare( | |||||
storage, | |||||
identifiers.CoreSWHID( | |||||
object_type=identifiers.ObjectType.SNAPSHOT, object_id=obj_id | |||||
), | |||||
fsck=fsck, | |||||
) as res: | |||||
yield res | |||||
@pytest.fixture( | |||||
scope="module", params=[cook_extract_snapshot_git_bare], | |||||
) | |||||
def cook_extract_snapshot(request): | |||||
"""Equivalent to cook_extract_snapshot_git_bare; but analogous to | |||||
cook_extract_revision in case we ever have more cookers supporting snapshots""" | |||||
return request.param | |||||
TEST_CONTENT = ( | TEST_CONTENT = ( | ||||
" test content\n" "and unicode \N{BLACK HEART SUIT}\n" " and trailing spaces " | " test content\n" "and unicode \N{BLACK HEART SUIT}\n" " and trailing spaces " | ||||
) | ) | ||||
TEST_EXECUTABLE = b"\x42\x40\x00\x00\x05" | TEST_EXECUTABLE = b"\x42\x40\x00\x00\x05" | ||||
class TestDirectoryCooker: | class TestDirectoryCooker: | ||||
def test_directory_simple(self, git_loader, cook_extract_directory): | def test_directory_simple(self, git_loader, cook_extract_directory): | ||||
▲ Show 20 Lines • Show All 188 Lines • ▼ Show 20 Lines | def test_directory_revision_data(self, swh_storage): | ||||
) | ) | ||||
swh_storage.directory_add([dir]) | swh_storage.directory_add([dir]) | ||||
with cook_extract_directory_dircooker(swh_storage, dir.id, fsck=False) as p: | with cook_extract_directory_dircooker(swh_storage, dir.id, fsck=False) as p: | ||||
assert (p / "submodule").is_symlink() | assert (p / "submodule").is_symlink() | ||||
assert os.readlink(str(p / "submodule")) == target_rev | assert os.readlink(str(p / "submodule")) == target_rev | ||||
class TestRevisionCooker: | class RepoFixtures: | ||||
def test_revision_simple(self, git_loader, cook_extract_revision): | """Shared loading and checking methods that can be reused by different types | ||||
of tests.""" | |||||
def load_repo_simple(self, git_loader): | |||||
# | # | ||||
# 1--2--3--4--5--6--7 | # 1--2--3--4--5--6--7 | ||||
# | # | ||||
repo = TestRepo() | repo = TestRepo() | ||||
with repo as rp: | with repo as rp: | ||||
(rp / "file1").write_text(TEST_CONTENT) | (rp / "file1").write_text(TEST_CONTENT) | ||||
repo.commit("add file1") | repo.commit("add file1") | ||||
(rp / "file2").write_text(TEST_CONTENT) | (rp / "file2").write_text(TEST_CONTENT) | ||||
Show All 9 Lines | def load_repo_simple(self, git_loader): | ||||
(rp / "file2").unlink() | (rp / "file2").unlink() | ||||
repo.commit("remove file2") | repo.commit("remove file2") | ||||
(rp / "bin1").rename(rp / "bin") | (rp / "bin1").rename(rp / "bin") | ||||
repo.commit("rename bin1 to bin") | repo.commit("rename bin1 to bin") | ||||
loader = git_loader(str(rp)) | loader = git_loader(str(rp)) | ||||
loader.load() | loader.load() | ||||
obj_id_hex = repo.repo.refs[b"HEAD"].decode() | obj_id_hex = repo.repo.refs[b"HEAD"].decode() | ||||
obj_id = hashutil.hash_to_bytes(obj_id_hex) | obj_id = hashutil.hash_to_bytes(obj_id_hex) | ||||
return (loader, obj_id) | |||||
with cook_extract_revision(loader.storage, obj_id) as (ert, p): | def check_revision_simple(self, ert, p, obj_id): | ||||
ert.checkout(b"HEAD") | ert.checkout(b"HEAD") | ||||
assert (p / "file1").stat().st_mode == 0o100644 | assert (p / "file1").stat().st_mode == 0o100644 | ||||
assert (p / "file1").read_text() == TEST_CONTENT | assert (p / "file1").read_text() == TEST_CONTENT | ||||
assert (p / "link1").is_symlink() | assert (p / "link1").is_symlink() | ||||
assert os.readlink(str(p / "link1")) == "file1" | assert os.readlink(str(p / "link1")) == "file1" | ||||
assert (p / "bin").stat().st_mode == 0o100755 | assert (p / "bin").stat().st_mode == 0o100755 | ||||
assert (p / "bin").read_bytes() == TEST_EXECUTABLE | assert (p / "bin").read_bytes() == TEST_EXECUTABLE | ||||
assert (p / "dir1/dir2/file").read_text() == TEST_CONTENT | assert (p / "dir1/dir2/file").read_text() == TEST_CONTENT | ||||
assert (p / "dir1/dir2/file").stat().st_mode == 0o100644 | assert (p / "dir1/dir2/file").stat().st_mode == 0o100644 | ||||
assert ert.repo.refs[b"HEAD"].decode() == obj_id_hex | assert ert.repo.refs[b"HEAD"].decode() == obj_id.hex() | ||||
def test_revision_two_roots(self, git_loader, cook_extract_revision): | def load_repo_two_roots(self, git_loader): | ||||
# | # | ||||
# 1----3---4 | # 1----3---4 | ||||
# / | # / | ||||
# 2---- | # 2---- | ||||
# | # | ||||
repo = TestRepo() | repo = TestRepo() | ||||
with repo as rp: | with repo as rp: | ||||
(rp / "file1").write_text(TEST_CONTENT) | (rp / "file1").write_text(TEST_CONTENT) | ||||
c1 = repo.commit("Add file1") | c1 = repo.commit("Add file1") | ||||
del repo.repo.refs[b"refs/heads/master"] # git update-ref -d HEAD | del repo.repo.refs[b"refs/heads/master"] # git update-ref -d HEAD | ||||
(rp / "file2").write_text(TEST_CONTENT) | (rp / "file2").write_text(TEST_CONTENT) | ||||
repo.commit("Add file2") | repo.commit("Add file2") | ||||
repo.merge([c1]) | repo.merge([c1]) | ||||
(rp / "file3").write_text(TEST_CONTENT) | (rp / "file3").write_text(TEST_CONTENT) | ||||
repo.commit("add file3") | repo.commit("add file3") | ||||
obj_id_hex = repo.repo.refs[b"HEAD"].decode() | obj_id_hex = repo.repo.refs[b"HEAD"].decode() | ||||
obj_id = hashutil.hash_to_bytes(obj_id_hex) | obj_id = hashutil.hash_to_bytes(obj_id_hex) | ||||
loader = git_loader(str(rp)) | loader = git_loader(str(rp)) | ||||
loader.load() | loader.load() | ||||
return (loader, obj_id) | |||||
with cook_extract_revision(loader.storage, obj_id) as (ert, p): | def check_revision_two_roots(self, ert, p, obj_id): | ||||
assert ert.repo.refs[b"HEAD"].decode() == obj_id_hex | assert ert.repo.refs[b"HEAD"].decode() == obj_id.hex() | ||||
def test_revision_two_double_fork_merge(self, git_loader, cook_extract_revision): | (c3,) = ert.repo[hashutil.hash_to_bytehex(obj_id)].parents | ||||
assert len(ert.repo[c3].parents) == 2 | |||||
def load_repo_two_heads(self, git_loader): | |||||
# | |||||
# 1---2----4 <-- master and b1 | |||||
# \ | |||||
# ----3 <-- b2 | |||||
# | |||||
repo = TestRepo() | |||||
with repo as rp: | |||||
(rp / "file1").write_text(TEST_CONTENT) | |||||
repo.commit("Add file1") | |||||
(rp / "file2").write_text(TEST_CONTENT) | |||||
c2 = repo.commit("Add file2") | |||||
repo.repo.refs[b"refs/heads/b2"] = c2 # branch b2 from master | |||||
(rp / "file3").write_text(TEST_CONTENT) | |||||
repo.commit("add file3", ref=b"refs/heads/b2") | |||||
(rp / "file4").write_text(TEST_CONTENT) | |||||
c4 = repo.commit("add file4", ref=b"refs/heads/master") | |||||
repo.repo.refs[b"refs/heads/b1"] = c4 # branch b1 from master | |||||
obj_id_hex = repo.repo.refs[b"HEAD"].decode() | |||||
obj_id = hashutil.hash_to_bytes(obj_id_hex) | |||||
loader = git_loader(str(rp)) | |||||
loader.load() | |||||
return (loader, obj_id) | |||||
def check_snapshot_two_heads(self, ert, p, obj_id): | |||||
assert ( | |||||
hashutil.hash_to_bytehex(obj_id) | |||||
== ert.repo.refs[b"HEAD"] | |||||
== ert.repo.refs[b"refs/heads/master"] | |||||
== ert.repo.refs[b"refs/remotes/origin/HEAD"] | |||||
== ert.repo.refs[b"refs/remotes/origin/master"] | |||||
== ert.repo.refs[b"refs/remotes/origin/b1"] | |||||
) | |||||
c4_id = hashutil.hash_to_bytehex(obj_id) | |||||
c3_id = ert.repo.refs[b"refs/remotes/origin/b2"] | |||||
assert ert.repo[c3_id].parents == ert.repo[c4_id].parents | |||||
def load_repo_two_double_fork_merge(self, git_loader): | |||||
# | # | ||||
# 2---4---6 | # 2---4---6 | ||||
# / / / | # / / / | ||||
# 1---3---5 | # 1---3---5 | ||||
# | # | ||||
repo = TestRepo() | repo = TestRepo() | ||||
with repo as rp: | with repo as rp: | ||||
(rp / "file1").write_text(TEST_CONTENT) | (rp / "file1").write_text(TEST_CONTENT) | ||||
c1 = repo.commit("Add file1") | c1 = repo.commit("Add file1") # create commit 1 | ||||
repo.repo.refs[b"refs/heads/c1"] = c1 | repo.repo.refs[b"refs/heads/c1"] = c1 # branch c1 from master | ||||
(rp / "file2").write_text(TEST_CONTENT) | (rp / "file2").write_text(TEST_CONTENT) | ||||
repo.commit("Add file2") | repo.commit("Add file2") # create commit 2 | ||||
(rp / "file3").write_text(TEST_CONTENT) | (rp / "file3").write_text(TEST_CONTENT) | ||||
c3 = repo.commit("Add file3", ref=b"refs/heads/c1") | c3 = repo.commit("Add file3", ref=b"refs/heads/c1") # create commit 3 on c1 | ||||
repo.repo.refs[b"refs/heads/c3"] = c3 | repo.repo.refs[b"refs/heads/c3"] = c3 # branch c3 from c1 | ||||
repo.merge([c3]) | repo.merge([c3]) # create commit 4 | ||||
(rp / "file5").write_text(TEST_CONTENT) | (rp / "file5").write_text(TEST_CONTENT) | ||||
c5 = repo.commit("Add file3", ref=b"refs/heads/c3") | c5 = repo.commit("Add file3", ref=b"refs/heads/c3") # create commit 5 on c3 | ||||
repo.merge([c5]) | repo.merge([c5]) # create commit 6 | ||||
obj_id_hex = repo.repo.refs[b"HEAD"].decode() | obj_id_hex = repo.repo.refs[b"HEAD"].decode() | ||||
obj_id = hashutil.hash_to_bytes(obj_id_hex) | obj_id = hashutil.hash_to_bytes(obj_id_hex) | ||||
loader = git_loader(str(rp)) | loader = git_loader(str(rp)) | ||||
loader.load() | loader.load() | ||||
return (loader, obj_id) | |||||
with cook_extract_revision(loader.storage, obj_id) as (ert, p): | def check_revision_two_double_fork_merge(self, ert, p, obj_id): | ||||
assert ert.repo.refs[b"HEAD"].decode() == obj_id_hex | assert ert.repo.refs[b"HEAD"].decode() == obj_id.hex() | ||||
def test_revision_triple_merge(self, git_loader, cook_extract_revision): | def check_snapshot_two_double_fork_merge(self, ert, p, obj_id): | ||||
assert ( | |||||
hashutil.hash_to_bytehex(obj_id) | |||||
== ert.repo.refs[b"HEAD"] | |||||
== ert.repo.refs[b"refs/heads/master"] | |||||
== ert.repo.refs[b"refs/remotes/origin/HEAD"] | |||||
== ert.repo.refs[b"refs/remotes/origin/master"] | |||||
) | |||||
(c4_id, c5_id) = ert.repo[obj_id.hex().encode()].parents | |||||
assert c5_id == ert.repo.refs[b"refs/remotes/origin/c3"] | |||||
(c2_id, c3_id) = ert.repo[c4_id].parents | |||||
assert c3_id == ert.repo.refs[b"refs/remotes/origin/c1"] | |||||
def load_repo_triple_merge(self, git_loader): | |||||
# | # | ||||
# .---.---5 | # .---.---5 | ||||
# / / / | # / / / | ||||
# 2 3 4 | # 2 3 4 | ||||
# / / / | # / / / | ||||
# 1---.---. | # 1---.---. | ||||
# | # | ||||
repo = TestRepo() | repo = TestRepo() | ||||
with repo as rp: | with repo as rp: | ||||
(rp / "file1").write_text(TEST_CONTENT) | (rp / "file1").write_text(TEST_CONTENT) | ||||
c1 = repo.commit("Commit 1") | c1 = repo.commit("Commit 1") | ||||
repo.repo.refs[b"refs/heads/b1"] = c1 | repo.repo.refs[b"refs/heads/b1"] = c1 | ||||
repo.repo.refs[b"refs/heads/b2"] = c1 | repo.repo.refs[b"refs/heads/b2"] = c1 | ||||
repo.commit("Commit 2") | repo.commit("Commit 2") | ||||
c3 = repo.commit("Commit 3", ref=b"refs/heads/b1") | c3 = repo.commit("Commit 3", ref=b"refs/heads/b1") | ||||
c4 = repo.commit("Commit 4", ref=b"refs/heads/b2") | c4 = repo.commit("Commit 4", ref=b"refs/heads/b2") | ||||
repo.merge([c3, c4]) | repo.merge([c3, c4]) | ||||
obj_id_hex = repo.repo.refs[b"HEAD"].decode() | obj_id_hex = repo.repo.refs[b"HEAD"].decode() | ||||
obj_id = hashutil.hash_to_bytes(obj_id_hex) | obj_id = hashutil.hash_to_bytes(obj_id_hex) | ||||
loader = git_loader(str(rp)) | loader = git_loader(str(rp)) | ||||
loader.load() | loader.load() | ||||
return (loader, obj_id) | |||||
with cook_extract_revision(loader.storage, obj_id) as (ert, p): | def check_revision_triple_merge(self, ert, p, obj_id): | ||||
assert ert.repo.refs[b"HEAD"].decode() == obj_id_hex | assert ert.repo.refs[b"HEAD"].decode() == obj_id.hex() | ||||
def test_revision_filtered_objects(self, git_loader, cook_extract_revision): | def check_snapshot_triple_merge(self, ert, p, obj_id): | ||||
assert ( | |||||
hashutil.hash_to_bytehex(obj_id) | |||||
== ert.repo.refs[b"HEAD"] | |||||
== ert.repo.refs[b"refs/heads/master"] | |||||
== ert.repo.refs[b"refs/remotes/origin/HEAD"] | |||||
== ert.repo.refs[b"refs/remotes/origin/master"] | |||||
) | |||||
(c2_id, c3_id, c4_id) = ert.repo[obj_id.hex().encode()].parents | |||||
assert c3_id == ert.repo.refs[b"refs/remotes/origin/b1"] | |||||
assert c4_id == ert.repo.refs[b"refs/remotes/origin/b2"] | |||||
assert ( | |||||
ert.repo[c2_id].parents | |||||
== ert.repo[c3_id].parents | |||||
== ert.repo[c4_id].parents | |||||
) | |||||
def load_repo_filtered_objects(self, git_loader): | |||||
repo = TestRepo() | repo = TestRepo() | ||||
with repo as rp: | with repo as rp: | ||||
file_1, id_1 = hash_content(b"test1") | file_1, id_1 = hash_content(b"test1") | ||||
file_2, id_2 = hash_content(b"test2") | file_2, id_2 = hash_content(b"test2") | ||||
file_3, id_3 = hash_content(b"test3") | file_3, id_3 = hash_content(b"test3") | ||||
(rp / "file").write_bytes(file_1) | (rp / "file").write_bytes(file_1) | ||||
(rp / "hidden_file").write_bytes(file_2) | (rp / "hidden_file").write_bytes(file_2) | ||||
Show All 26 Lines | def load_repo_filtered_objects(self, git_loader): | ||||
select sha1, sha1_git, sha256, blake2s256, length, 'no reason' | select sha1, sha1_git, sha256, blake2s256, length, 'no reason' | ||||
from content | from content | ||||
where sha1 = %s | where sha1 = %s | ||||
""", | """, | ||||
(id_3,), | (id_3,), | ||||
) | ) | ||||
cur.execute("delete from content where sha1 = %s", (id_3,)) | cur.execute("delete from content where sha1 = %s", (id_3,)) | ||||
return (loader, obj_id) | |||||
with cook_extract_revision(loader.storage, obj_id) as (ert, p): | def check_revision_filtered_objects(self, ert, p, obj_id): | ||||
ert.checkout(b"HEAD") | ert.checkout(b"HEAD") | ||||
assert (p / "file").read_bytes() == b"test1" | assert (p / "file").read_bytes() == b"test1" | ||||
assert (p / "hidden_file").read_bytes() == HIDDEN_MESSAGE | assert (p / "hidden_file").read_bytes() == HIDDEN_MESSAGE | ||||
assert (p / "absent_file").read_bytes() == SKIPPED_MESSAGE | assert (p / "absent_file").read_bytes() == SKIPPED_MESSAGE | ||||
def test_revision_null_fields(self, git_loader, cook_extract_revision): | def load_repo_null_fields(self, git_loader): | ||||
# Our schema doesn't enforce a lot of non-null revision fields. We need | # Our schema doesn't enforce a lot of non-null revision fields. We need | ||||
# to check these cases don't break the cooker. | # to check these cases don't break the cooker. | ||||
repo = TestRepo() | repo = TestRepo() | ||||
with repo as rp: | with repo as rp: | ||||
(rp / "file").write_text(TEST_CONTENT) | (rp / "file").write_text(TEST_CONTENT) | ||||
c = repo.commit("initial commit") | c = repo.commit("initial commit") | ||||
loader = git_loader(str(rp)) | loader = git_loader(str(rp)) | ||||
loader.load() | loader.load() | ||||
Show All 11 Lines | def load_repo_null_fields(self, git_loader): | ||||
type=RevisionType.GIT, | type=RevisionType.GIT, | ||||
directory=dir_id, | directory=dir_id, | ||||
metadata={}, | metadata={}, | ||||
synthetic=True, | synthetic=True, | ||||
) | ) | ||||
storage = loader.storage | storage = loader.storage | ||||
storage.revision_add([test_revision]) | storage.revision_add([test_revision]) | ||||
return (loader, test_revision.id) | |||||
with cook_extract_revision(storage, test_revision.id, fsck=False) as (ert, p): | def check_revision_null_fields(self, ert, p, obj_id): | ||||
ert.checkout(b"HEAD") | ert.checkout(b"HEAD") | ||||
assert (p / "file").stat().st_mode == 0o100644 | assert (p / "file").stat().st_mode == 0o100644 | ||||
def load_repo_tags(self, git_loader): | |||||
# v-- t2 | |||||
# | |||||
# 1---2----5 <-- master, t5, and t5a (annotated) | |||||
# \ | |||||
# ----3----4 <-- t4a (annotated) | |||||
# | |||||
repo = TestRepo() | |||||
with repo as rp: | |||||
(rp / "file1").write_text(TEST_CONTENT) | |||||
repo.commit("Add file1") | |||||
(rp / "file2").write_text(TEST_CONTENT) | |||||
repo.commit("Add file2") # create c2 | |||||
repo.tag(b"t2") | |||||
(rp / "file3").write_text(TEST_CONTENT) | |||||
repo.commit("add file3") | |||||
(rp / "file4").write_text(TEST_CONTENT) | |||||
repo.commit("add file4") | |||||
repo.tag(b"t4a", message=b"tag 4") | |||||
# Go back to c2 | |||||
repo.git_shell("reset", "--hard", "HEAD^^") | |||||
(rp / "file5").write_text(TEST_CONTENT) | |||||
repo.commit("add file5") # create c5 | |||||
repo.tag(b"t5") | |||||
repo.tag(b"t5a", message=b"tag 5") | |||||
obj_id_hex = repo.repo.refs[b"HEAD"].decode() | |||||
obj_id = hashutil.hash_to_bytes(obj_id_hex) | |||||
loader = git_loader(str(rp)) | |||||
loader.load() | |||||
return (loader, obj_id) | |||||
def check_snapshot_tags(self, ert, p, obj_id): | |||||
assert ( | |||||
hashutil.hash_to_bytehex(obj_id) | |||||
== ert.repo.refs[b"HEAD"] | |||||
== ert.repo.refs[b"refs/heads/master"] | |||||
== ert.repo.refs[b"refs/remotes/origin/HEAD"] | |||||
== ert.repo.refs[b"refs/remotes/origin/master"] | |||||
== ert.repo.refs[b"refs/tags/t5"] | |||||
) | |||||
c2_id = ert.repo.refs[b"refs/tags/t2"] | |||||
c5_id = hashutil.hash_to_bytehex(obj_id) | |||||
assert ert.repo[c5_id].parents == [c2_id] | |||||
t5a = ert.repo[ert.repo.refs[b"refs/tags/t5a"]] | |||||
assert t5a.message == b"tag 5" | |||||
assert t5a.object == (dulwich.objects.Commit, c5_id) | |||||
t4a = ert.repo[ert.repo.refs[b"refs/tags/t4a"]] | |||||
(_, c4_id) = t4a.object | |||||
assert ert.repo[c4_id].message == b"add file4\n" | |||||
(c3_id,) = ert.repo[c4_id].parents | |||||
assert ert.repo[c3_id].message == b"add file3\n" | |||||
assert ert.repo[c3_id].parents == [c2_id] | |||||
class TestRevisionCooker(RepoFixtures): | |||||
def test_revision_simple(self, git_loader, cook_extract_revision): | |||||
(loader, obj_id) = self.load_repo_simple(git_loader) | |||||
with cook_extract_revision(loader.storage, obj_id) as (ert, p): | |||||
self.check_revision_simple(ert, p, obj_id) | |||||
def test_revision_two_roots(self, git_loader, cook_extract_revision): | |||||
(loader, obj_id) = self.load_repo_two_roots(git_loader) | |||||
with cook_extract_revision(loader.storage, obj_id) as (ert, p): | |||||
self.check_revision_two_roots(ert, p, obj_id) | |||||
def test_revision_two_double_fork_merge(self, git_loader, cook_extract_revision): | |||||
(loader, obj_id) = self.load_repo_two_double_fork_merge(git_loader) | |||||
with cook_extract_revision(loader.storage, obj_id) as (ert, p): | |||||
self.check_revision_two_double_fork_merge(ert, p, obj_id) | |||||
def test_revision_triple_merge(self, git_loader, cook_extract_revision): | |||||
(loader, obj_id) = self.load_repo_triple_merge(git_loader) | |||||
with cook_extract_revision(loader.storage, obj_id) as (ert, p): | |||||
self.check_revision_triple_merge(ert, p, obj_id) | |||||
def test_revision_filtered_objects(self, git_loader, cook_extract_revision): | |||||
(loader, obj_id) = self.load_repo_filtered_objects(git_loader) | |||||
with cook_extract_revision(loader.storage, obj_id) as (ert, p): | |||||
self.check_revision_filtered_objects(ert, p, obj_id) | |||||
def test_revision_null_fields(self, git_loader, cook_extract_revision): | |||||
(loader, obj_id) = self.load_repo_null_fields(git_loader) | |||||
with cook_extract_revision(loader.storage, obj_id, fsck=False) as (ert, p): | |||||
self.check_revision_null_fields(ert, p, obj_id) | |||||
def test_revision_revision_data(self, swh_storage): | def test_revision_revision_data(self, swh_storage): | ||||
target_rev = "0e8a3ad980ec179856012b7eecf4327e99cd44cd" | target_rev = "0e8a3ad980ec179856012b7eecf4327e99cd44cd" | ||||
dir = Directory( | dir = Directory( | ||||
entries=( | entries=( | ||||
DirectoryEntry( | DirectoryEntry( | ||||
name=b"submodule", | name=b"submodule", | ||||
type="rev", | type="rev", | ||||
Show All 16 Lines | def test_revision_revision_data(self, swh_storage): | ||||
metadata={}, | metadata={}, | ||||
synthetic=True, | synthetic=True, | ||||
) | ) | ||||
swh_storage.revision_add([rev]) | swh_storage.revision_add([rev]) | ||||
with cook_stream_revision_gitfast(swh_storage, rev.id) as stream: | with cook_stream_revision_gitfast(swh_storage, rev.id) as stream: | ||||
pattern = "M 160000 {} submodule".format(target_rev).encode() | pattern = "M 160000 {} submodule".format(target_rev).encode() | ||||
assert pattern in stream.read() | assert pattern in stream.read() | ||||
class TestSnapshotCooker(RepoFixtures): | |||||
def test_snapshot_simple(self, git_loader, cook_extract_snapshot): | |||||
(loader, main_rev_id) = self.load_repo_simple(git_loader) | |||||
snp_id = loader.loaded_snapshot_id | |||||
with cook_extract_snapshot(loader.storage, snp_id) as (ert, p): | |||||
self.check_revision_simple(ert, p, main_rev_id) | |||||
def test_snapshot_two_roots(self, git_loader, cook_extract_snapshot): | |||||
(loader, main_rev_id) = self.load_repo_two_roots(git_loader) | |||||
snp_id = loader.loaded_snapshot_id | |||||
with cook_extract_snapshot(loader.storage, snp_id) as (ert, p): | |||||
self.check_revision_two_roots(ert, p, main_rev_id) | |||||
def test_snapshot_two_heads(self, git_loader, cook_extract_snapshot): | |||||
(loader, main_rev_id) = self.load_repo_two_heads(git_loader) | |||||
snp_id = loader.loaded_snapshot_id | |||||
with cook_extract_snapshot(loader.storage, snp_id) as (ert, p): | |||||
self.check_snapshot_two_heads(ert, p, main_rev_id) | |||||
def test_snapshot_two_double_fork_merge(self, git_loader, cook_extract_snapshot): | |||||
(loader, main_rev_id) = self.load_repo_two_double_fork_merge(git_loader) | |||||
snp_id = loader.loaded_snapshot_id | |||||
with cook_extract_snapshot(loader.storage, snp_id) as (ert, p): | |||||
self.check_revision_two_double_fork_merge(ert, p, main_rev_id) | |||||
self.check_snapshot_two_double_fork_merge(ert, p, main_rev_id) | |||||
def test_snapshot_triple_merge(self, git_loader, cook_extract_snapshot): | |||||
(loader, main_rev_id) = self.load_repo_triple_merge(git_loader) | |||||
snp_id = loader.loaded_snapshot_id | |||||
with cook_extract_snapshot(loader.storage, snp_id) as (ert, p): | |||||
self.check_revision_triple_merge(ert, p, main_rev_id) | |||||
self.check_snapshot_triple_merge(ert, p, main_rev_id) | |||||
def test_snapshot_filtered_objects(self, git_loader, cook_extract_snapshot): | |||||
(loader, main_rev_id) = self.load_repo_filtered_objects(git_loader) | |||||
snp_id = loader.loaded_snapshot_id | |||||
with cook_extract_snapshot(loader.storage, snp_id) as (ert, p): | |||||
self.check_revision_filtered_objects(ert, p, main_rev_id) | |||||
def test_snapshot_tags(self, git_loader, cook_extract_snapshot): | |||||
(loader, main_rev_id) = self.load_repo_tags(git_loader) | |||||
snp_id = loader.loaded_snapshot_id | |||||
with cook_extract_snapshot(loader.storage, snp_id) as (ert, p): | |||||
self.check_snapshot_tags(ert, p, main_rev_id) |