diff --git a/dulwich/fastexport.py b/dulwich/fastexport.py index 28e487c7..630659eb 100644 --- a/dulwich/fastexport.py +++ b/dulwich/fastexport.py @@ -1,232 +1,234 @@ # __init__.py -- Fast export/import functionality # Copyright (C) 2010-2013 Jelmer Vernooij # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; version 2 # of the License or (at your option) any later version of # the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # MA 02110-1301, USA. """Fast export/import functionality.""" import sys from dulwich.index import ( commit_tree, ) from dulwich.objects import ( Blob, Commit, Tag, ) from fastimport import __version__ as fastimport_version if fastimport_version <= (0, 9, 5) and sys.version_info[0] == 3 and sys.version_info[1] < 5: raise ImportError("Older versions of fastimport don't support python3<3.5") from fastimport import ( commands, errors as fastimport_errors, parser, processor, ) import stat def split_email(text): (name, email) = text.rsplit(b" <", 1) return (name, email.rstrip(b">")) class GitFastExporter(object): """Generate a fast-export output stream for Git objects.""" def __init__(self, outf, store): self.outf = outf self.store = store self.markers = {} self._marker_idx = 0 def print_cmd(self, cmd): self.outf.write(getattr(cmd, "__bytes__", cmd.__repr__)() + b"\n") def _allocate_marker(self): self._marker_idx+=1 return b"%d" % (self._marker_idx,) def _export_blob(self, blob): marker = self._allocate_marker() self.markers[marker] = blob.id return (commands.BlobCommand(marker, blob.data), marker) def emit_blob(self, blob): (cmd, marker) = self._export_blob(blob) self.print_cmd(cmd) return marker def _iter_files(self, base_tree, new_tree): for ((old_path, new_path), (old_mode, new_mode), (old_hexsha, new_hexsha)) in \ self.store.tree_changes(base_tree, new_tree): if new_path is None: yield commands.FileDeleteCommand(old_path) continue if not stat.S_ISDIR(new_mode): blob = self.store[new_hexsha] marker = self.emit_blob(blob) if old_path != new_path and old_path is not None: yield commands.FileRenameCommand(old_path, new_path) if old_mode != new_mode or old_hexsha != new_hexsha: - yield commands.FileModifyCommand(new_path, new_mode, marker, - None) + prefixed_marker = b':%s' % (marker,) + yield commands.FileModifyCommand( + new_path, new_mode, prefixed_marker, None + ) def _export_commit(self, commit, ref, base_tree=None): file_cmds = list(self._iter_files(base_tree, commit.tree)) marker = self._allocate_marker() if commit.parents: from_ = commit.parents[0] merges = commit.parents[1:] else: from_ = None merges = [] author, author_email = split_email(commit.author) committer, committer_email = split_email(commit.committer) cmd = commands.CommitCommand(ref, marker, (author, author_email, commit.author_time, commit.author_timezone), (committer, committer_email, commit.commit_time, commit.commit_timezone), commit.message, from_, merges, file_cmds) return (cmd, marker) def emit_commit(self, commit, ref, base_tree=None): cmd, marker = self._export_commit(commit, ref, base_tree) self.print_cmd(cmd) return marker class GitImportProcessor(processor.ImportProcessor): """An import processor that imports into a Git repository using Dulwich. """ # FIXME: Batch creation of objects? def __init__(self, repo, params=None, verbose=False, outf=None): processor.ImportProcessor.__init__(self, params, verbose) self.repo = repo self.last_commit = None self.markers = {} self._contents = {} def import_stream(self, stream): p = parser.ImportParser(stream) self.process(p.iter_commands) return self.markers def blob_handler(self, cmd): """Process a BlobCommand.""" blob = Blob.from_string(cmd.data) self.repo.object_store.add_object(blob) if cmd.mark: self.markers[cmd.mark] = blob.id def checkpoint_handler(self, cmd): """Process a CheckpointCommand.""" pass def commit_handler(self, cmd): """Process a CommitCommand.""" commit = Commit() if cmd.author is not None: author = cmd.author else: author = cmd.committer (author_name, author_email, author_timestamp, author_timezone) = author (committer_name, committer_email, commit_timestamp, commit_timezone) = cmd.committer commit.author = author_name + b" <" + author_email + b">" commit.author_timezone = author_timezone commit.author_time = int(author_timestamp) commit.committer = committer_name + b" <" + committer_email + b">" commit.commit_timezone = commit_timezone commit.commit_time = int(commit_timestamp) commit.message = cmd.message commit.parents = [] if cmd.from_: self._reset_base(cmd.from_) for filecmd in cmd.iter_files(): if filecmd.name == b"filemodify": if filecmd.data is not None: blob = Blob.from_string(filecmd.data) self.repo.object_store.add(blob) blob_id = blob.id else: assert filecmd.dataref.startswith(b":"), \ "non-marker refs not supported yet (%r)" % filecmd.dataref blob_id = self.markers[filecmd.dataref[1:]] self._contents[filecmd.path] = (filecmd.mode, blob_id) elif filecmd.name == b"filedelete": del self._contents[filecmd.path] elif filecmd.name == b"filecopy": self._contents[filecmd.dest_path] = self._contents[ filecmd.src_path] elif filecmd.name == b"filerename": self._contents[filecmd.new_path] = self._contents[ filecmd.old_path] del self._contents[filecmd.old_path] elif filecmd.name == b"filedeleteall": self._contents = {} else: raise Exception("Command %s not supported" % filecmd.name) commit.tree = commit_tree(self.repo.object_store, ((path, hexsha, mode) for (path, (mode, hexsha)) in self._contents.items())) if self.last_commit is not None: commit.parents.append(self.last_commit) commit.parents += cmd.merges self.repo.object_store.add_object(commit) self.repo[cmd.ref] = commit.id self.last_commit = commit.id if cmd.mark: self.markers[cmd.mark] = commit.id def progress_handler(self, cmd): """Process a ProgressCommand.""" pass def _reset_base(self, commit_id): if self.last_commit == commit_id: return self.last_commit = commit_id self._contents = {} tree_id = self.repo[commit_id].tree for (path, mode, hexsha) in ( self.repo.object_store.iter_tree_contents(tree_id)): self._contents[path] = (mode, hexsha) def reset_handler(self, cmd): """Process a ResetCommand.""" self._reset_base(cmd.from_) self.repo.refs[cmd.ref] = cmd.from_ def tag_handler(self, cmd): """Process a TagCommand.""" tag = Tag() tag.tagger = cmd.tagger tag.message = cmd.message tag.name = cmd.tag self.repo.add_object(tag) self.repo.refs["refs/tags/" + tag.name] = tag.id def feature_handler(self, cmd): """Process a FeatureCommand.""" raise fastimport_errors.UnknownFeature(cmd.feature_name) diff --git a/dulwich/tests/test_fastexport.py b/dulwich/tests/test_fastexport.py index 95652537..1a447800 100644 --- a/dulwich/tests/test_fastexport.py +++ b/dulwich/tests/test_fastexport.py @@ -1,213 +1,213 @@ # test_fastexport.py -- Fast export/import functionality # Copyright (C) 2010 Jelmer Vernooij # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; version 2 # of the License or (at your option) any later version of # the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # MA 02110-1301, USA. from io import BytesIO import stat from dulwich.object_store import ( MemoryObjectStore, ) from dulwich.objects import ( Blob, Commit, Tree, ) from dulwich.repo import ( MemoryRepo, ) from dulwich.tests import ( SkipTest, TestCase, ) from dulwich.tests.utils import ( build_commit_graph, ) class GitFastExporterTests(TestCase): """Tests for the GitFastExporter tests.""" def setUp(self): super(GitFastExporterTests, self).setUp() self.store = MemoryObjectStore() self.stream = BytesIO() try: from dulwich.fastexport import GitFastExporter except ImportError: raise SkipTest("python-fastimport not available") self.fastexporter = GitFastExporter(self.stream, self.store) def test_emit_blob(self): b = Blob() b.data = b"fooBAR" self.fastexporter.emit_blob(b) self.assertEqual(b'blob\nmark :1\ndata 6\nfooBAR\n', self.stream.getvalue()) def test_emit_commit(self): b = Blob() b.data = b"FOO" t = Tree() t.add(b"foo", stat.S_IFREG | 0o644, b.id) c = Commit() c.committer = c.author = b"Jelmer " c.author_time = c.commit_time = 1271345553 c.author_timezone = c.commit_timezone = 0 c.message = b"msg" c.tree = t.id self.store.add_objects([(b, None), (t, None), (c, None)]) self.fastexporter.emit_commit(c, b"refs/heads/master") self.assertEqual(b"""blob mark :1 data 3 FOO commit refs/heads/master mark :2 author Jelmer 1271345553 +0000 committer Jelmer 1271345553 +0000 data 3 msg -M 644 1 foo +M 644 :1 foo """, self.stream.getvalue()) class GitImportProcessorTests(TestCase): """Tests for the GitImportProcessor tests.""" def setUp(self): super(GitImportProcessorTests, self).setUp() self.repo = MemoryRepo() try: from dulwich.fastexport import GitImportProcessor except ImportError: raise SkipTest("python-fastimport not available") self.processor = GitImportProcessor(self.repo) def test_reset_handler(self): from fastimport import commands [c1] = build_commit_graph(self.repo.object_store, [[1]]) cmd = commands.ResetCommand(b"refs/heads/foo", c1.id) self.processor.reset_handler(cmd) self.assertEqual(c1.id, self.repo.get_refs()[b"refs/heads/foo"]) def test_commit_handler(self): from fastimport import commands cmd = commands.CommitCommand(b"refs/heads/foo", b"mrkr", (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600), (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600), b"FOO", None, [], []) self.processor.commit_handler(cmd) commit = self.repo[self.processor.last_commit] self.assertEqual(b"Jelmer ", commit.author) self.assertEqual(b"Jelmer ", commit.committer) self.assertEqual(b"FOO", commit.message) self.assertEqual([], commit.parents) self.assertEqual(432432432.0, commit.commit_time) self.assertEqual(432432432.0, commit.author_time) self.assertEqual(3600, commit.commit_timezone) self.assertEqual(3600, commit.author_timezone) self.assertEqual(commit, self.repo[b"refs/heads/foo"]) def test_import_stream(self): markers = self.processor.import_stream(BytesIO(b"""blob mark :1 data 11 text for a commit refs/heads/master mark :2 committer Joe Foo 1288287382 +0000 data 20 M 100644 :1 a """)) self.assertEqual(2, len(markers)) self.assertTrue(isinstance(self.repo[markers[b"1"]], Blob)) self.assertTrue(isinstance(self.repo[markers[b"2"]], Commit)) def test_file_add(self): from fastimport import commands cmd = commands.BlobCommand(b"23", b"data") self.processor.blob_handler(cmd) cmd = commands.CommitCommand(b"refs/heads/foo", b"mrkr", (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600), (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600), b"FOO", None, [], [commands.FileModifyCommand(b"path", 0o100644, b":23", None)]) self.processor.commit_handler(cmd) commit = self.repo[self.processor.last_commit] self.assertEqual([ (b'path', 0o100644, b'6320cd248dd8aeaab759d5871f8781b5c0505172')], self.repo[commit.tree].items()) def simple_commit(self): from fastimport import commands cmd = commands.BlobCommand(b"23", b"data") self.processor.blob_handler(cmd) cmd = commands.CommitCommand(b"refs/heads/foo", b"mrkr", (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600), (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600), b"FOO", None, [], [commands.FileModifyCommand(b"path", 0o100644, b":23", None)]) self.processor.commit_handler(cmd) commit = self.repo[self.processor.last_commit] return commit def make_file_commit(self, file_cmds): """Create a trivial commit with the specified file commands. :param file_cmds: File commands to run. :return: The created commit object """ from fastimport import commands cmd = commands.CommitCommand(b"refs/heads/foo", b"mrkr", (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600), (b"Jelmer", b"jelmer@samba.org", 432432432.0, 3600), b"FOO", None, [], file_cmds) self.processor.commit_handler(cmd) return self.repo[self.processor.last_commit] def test_file_copy(self): from fastimport import commands self.simple_commit() commit = self.make_file_commit([commands.FileCopyCommand(b"path", b"new_path")]) self.assertEqual([ (b'new_path', 0o100644, b'6320cd248dd8aeaab759d5871f8781b5c0505172'), (b'path', 0o100644, b'6320cd248dd8aeaab759d5871f8781b5c0505172'), ], self.repo[commit.tree].items()) def test_file_move(self): from fastimport import commands self.simple_commit() commit = self.make_file_commit([commands.FileRenameCommand(b"path", b"new_path")]) self.assertEqual([ (b'new_path', 0o100644, b'6320cd248dd8aeaab759d5871f8781b5c0505172'), ], self.repo[commit.tree].items()) def test_file_delete(self): from fastimport import commands self.simple_commit() commit = self.make_file_commit([commands.FileDeleteCommand(b"path")]) self.assertEqual([], self.repo[commit.tree].items()) def test_file_deleteall(self): from fastimport import commands self.simple_commit() commit = self.make_file_commit([commands.FileDeleteAllCommand()]) self.assertEqual([], self.repo[commit.tree].items())