diff --git a/dulwich/protocol.py b/dulwich/protocol.py index 7328294a..96acf895 100644 --- a/dulwich/protocol.py +++ b/dulwich/protocol.py @@ -1,549 +1,549 @@ # protocol.py -- Shared parts of the git protocols # Copyright (C) 2008 John Carr # Copyright (C) 2008-2012 Jelmer Vernooij # # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU # General Public License as public by the Free Software Foundation; version 2.0 # or (at your option) any later version. You can redistribute it and/or # modify it under the terms of either of these two licenses. # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # You should have received a copy of the licenses; if not, see # for a copy of the GNU General Public License # and for a copy of the Apache # License, Version 2.0. # """Generic functions for talking the git smart server protocol.""" from io import BytesIO from os import ( SEEK_END, ) import socket import dulwich from dulwich.errors import ( HangupException, GitProtocolError, ) TCP_GIT_PORT = 9418 ZERO_SHA = b"0" * 40 SINGLE_ACK = 0 MULTI_ACK = 1 MULTI_ACK_DETAILED = 2 # pack data SIDE_BAND_CHANNEL_DATA = 1 # progress messages SIDE_BAND_CHANNEL_PROGRESS = 2 # fatal error message just before stream aborts SIDE_BAND_CHANNEL_FATAL = 3 CAPABILITY_DEEPEN_SINCE = b'deepen-since' CAPABILITY_DEEPEN_NOT = b'deepen-not' CAPABILITY_DEEPEN_RELATIVE = b'deepen-relative' CAPABILITY_DELETE_REFS = b'delete-refs' CAPABILITY_INCLUDE_TAG = b'include-tag' CAPABILITY_MULTI_ACK = b'multi_ack' CAPABILITY_MULTI_ACK_DETAILED = b'multi_ack_detailed' CAPABILITY_NO_DONE = b'no-done' CAPABILITY_NO_PROGRESS = b'no-progress' CAPABILITY_OFS_DELTA = b'ofs-delta' CAPABILITY_QUIET = b'quiet' CAPABILITY_REPORT_STATUS = b'report-status' CAPABILITY_SHALLOW = b'shallow' CAPABILITY_SIDE_BAND = b'side-band' CAPABILITY_SIDE_BAND_64K = b'side-band-64k' CAPABILITY_THIN_PACK = b'thin-pack' CAPABILITY_AGENT = b'agent' CAPABILITY_SYMREF = b'symref' # Magic ref that is used to attach capabilities to when # there are no refs. Should always be ste to ZERO_SHA. CAPABILITIES_REF = b'capabilities^{}' COMMON_CAPABILITIES = [ CAPABILITY_OFS_DELTA, CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K, CAPABILITY_AGENT, CAPABILITY_NO_PROGRESS] KNOWN_UPLOAD_CAPABILITIES = set(COMMON_CAPABILITIES + [ CAPABILITY_THIN_PACK, CAPABILITY_MULTI_ACK, CAPABILITY_MULTI_ACK_DETAILED, CAPABILITY_INCLUDE_TAG, CAPABILITY_DEEPEN_SINCE, CAPABILITY_SYMREF, CAPABILITY_SHALLOW, CAPABILITY_DEEPEN_NOT, CAPABILITY_DEEPEN_RELATIVE, ]) KNOWN_RECEIVE_CAPABILITIES = set(COMMON_CAPABILITIES + [ CAPABILITY_REPORT_STATUS]) def agent_string(): return ('dulwich/%d.%d.%d' % dulwich.__version__).encode('ascii') def capability_agent(): return CAPABILITY_AGENT + b'=' + agent_string() def capability_symref(from_ref, to_ref): return CAPABILITY_SYMREF + b'=' + from_ref + b':' + to_ref def extract_capability_names(capabilities): return set(parse_capability(c)[0] for c in capabilities) def parse_capability(capability): - parts = capability.split('=', 1) + parts = capability.split(b'=', 1) if len(parts) == 1: return (parts[0], None) return tuple(parts) def symref_capabilities(symrefs): return [capability_symref(*k) for k in symrefs] COMMAND_DEEPEN = b'deepen' COMMAND_SHALLOW = b'shallow' COMMAND_UNSHALLOW = b'unshallow' COMMAND_DONE = b'done' COMMAND_WANT = b'want' COMMAND_HAVE = b'have' class ProtocolFile(object): """A dummy file for network ops that expect file-like objects.""" def __init__(self, read, write): self.read = read self.write = write def tell(self): pass def close(self): pass def pkt_line(data): """Wrap data in a pkt-line. :param data: The data to wrap, as a str or None. :return: The data prefixed with its length in pkt-line format; if data was None, returns the flush-pkt ('0000'). """ if data is None: return b'0000' return ('%04x' % (len(data) + 4)).encode('ascii') + data class Protocol(object): """Class for interacting with a remote git process over the wire. Parts of the git wire protocol use 'pkt-lines' to communicate. A pkt-line consists of the length of the line as a 4-byte hex string, followed by the payload data. The length includes the 4-byte header. The special line '0000' indicates the end of a section of input and is called a 'flush-pkt'. For details on the pkt-line format, see the cgit distribution: Documentation/technical/protocol-common.txt """ def __init__(self, read, write, close=None, report_activity=None): self.read = read self.write = write self._close = close self.report_activity = report_activity self._readahead = None def close(self): if self._close: self._close() def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() def read_pkt_line(self): """Reads a pkt-line from the remote git process. This method may read from the readahead buffer; see unread_pkt_line. :return: The next string from the stream, without the length prefix, or None for a flush-pkt ('0000'). """ if self._readahead is None: read = self.read else: read = self._readahead.read self._readahead = None try: sizestr = read(4) if not sizestr: raise HangupException() size = int(sizestr, 16) if size == 0: if self.report_activity: self.report_activity(4, 'read') return None if self.report_activity: self.report_activity(size, 'read') pkt_contents = read(size-4) except socket.error as e: raise GitProtocolError(e) else: if len(pkt_contents) + 4 != size: raise GitProtocolError( 'Length of pkt read %04x does not match length prefix %04x' % (len(pkt_contents) + 4, size)) return pkt_contents def eof(self): """Test whether the protocol stream has reached EOF. Note that this refers to the actual stream EOF and not just a flush-pkt. :return: True if the stream is at EOF, False otherwise. """ try: next_line = self.read_pkt_line() except HangupException: return True self.unread_pkt_line(next_line) return False def unread_pkt_line(self, data): """Unread a single line of data into the readahead buffer. This method can be used to unread a single pkt-line into a fixed readahead buffer. :param data: The data to unread, without the length prefix. :raise ValueError: If more than one pkt-line is unread. """ if self._readahead is not None: raise ValueError('Attempted to unread multiple pkt-lines.') self._readahead = BytesIO(pkt_line(data)) def read_pkt_seq(self): """Read a sequence of pkt-lines from the remote git process. :return: Yields each line of data up to but not including the next flush-pkt. """ pkt = self.read_pkt_line() while pkt: yield pkt pkt = self.read_pkt_line() def write_pkt_line(self, line): """Sends a pkt-line to the remote git process. :param line: A string containing the data to send, without the length prefix. """ try: line = pkt_line(line) self.write(line) if self.report_activity: self.report_activity(len(line), 'write') except socket.error as e: raise GitProtocolError(e) def write_file(self): """Return a writable file-like object for this protocol.""" class ProtocolFile(object): def __init__(self, proto): self._proto = proto self._offset = 0 def write(self, data): self._proto.write(data) self._offset += len(data) def tell(self): return self._offset def close(self): pass return ProtocolFile(self) def write_sideband(self, channel, blob): """Write multiplexed data to the sideband. :param channel: An int specifying the channel to write to. :param blob: A blob of data (as a string) to send on this channel. """ # a pktline can be a max of 65520. a sideband line can therefore be # 65520-5 = 65515 # WTF: Why have the len in ASCII, but the channel in binary. while blob: self.write_pkt_line(bytes(bytearray([channel])) + blob[:65515]) blob = blob[65515:] def send_cmd(self, cmd, *args): """Send a command and some arguments to a git server. Only used for the TCP git protocol (git://). :param cmd: The remote service to access. :param args: List of arguments to send to remove service. """ self.write_pkt_line(cmd + b" " + b"".join([(a + b"\0") for a in args])) def read_cmd(self): """Read a command and some arguments from the git client Only used for the TCP git protocol (git://). :return: A tuple of (command, [list of arguments]). """ line = self.read_pkt_line() splice_at = line.find(b" ") cmd, args = line[:splice_at], line[splice_at+1:] assert args[-1:] == b"\x00" return cmd, args[:-1].split(b"\0") _RBUFSIZE = 8192 # Default read buffer size. class ReceivableProtocol(Protocol): """Variant of Protocol that allows reading up to a size without blocking. This class has a recv() method that behaves like socket.recv() in addition to a read() method. If you want to read n bytes from the wire and block until exactly n bytes (or EOF) are read, use read(n). If you want to read at most n bytes from the wire but don't care if you get less, use recv(n). Note that recv(n) will still block until at least one byte is read. """ def __init__(self, recv, write, report_activity=None, rbufsize=_RBUFSIZE): super(ReceivableProtocol, self).__init__(self.read, write, report_activity) self._recv = recv self._rbuf = BytesIO() self._rbufsize = rbufsize def read(self, size): # From _fileobj.read in socket.py in the Python 2.6.5 standard library, # with the following modifications: # - omit the size <= 0 branch # - seek back to start rather than 0 in case some buffer has been # consumed. # - use SEEK_END instead of the magic number. # Copyright (c) 2001-2010 Python Software Foundation; All Rights # Reserved # Licensed under the Python Software Foundation License. # TODO: see if buffer is more efficient than cBytesIO. assert size > 0 # Our use of BytesIO rather than lists of string objects returned by # recv() minimizes memory usage and fragmentation that occurs when # rbufsize is large compared to the typical return value of recv(). buf = self._rbuf start = buf.tell() buf.seek(0, SEEK_END) # buffer may have been partially consumed by recv() buf_len = buf.tell() - start if buf_len >= size: # Already have size bytes in our buffer? Extract and return. buf.seek(start) rv = buf.read(size) self._rbuf = BytesIO() self._rbuf.write(buf.read()) self._rbuf.seek(0) return rv self._rbuf = BytesIO() # reset _rbuf. we consume it via buf. while True: left = size - buf_len # recv() will malloc the amount of memory given as its # parameter even though it often returns much less data # than that. The returned data string is short lived # as we copy it into a BytesIO and free it. This avoids # fragmentation issues on many platforms. data = self._recv(left) if not data: break n = len(data) if n == size and not buf_len: # Shortcut. Avoid buffer data copies when: # - We have no data in our buffer. # AND # - Our call to recv returned exactly the # number of bytes we were asked to read. return data if n == left: buf.write(data) del data # explicit free break assert n <= left, "_recv(%d) returned %d bytes" % (left, n) buf.write(data) buf_len += n del data # explicit free # assert buf_len == buf.tell() buf.seek(start) return buf.read() def recv(self, size): assert size > 0 buf = self._rbuf start = buf.tell() buf.seek(0, SEEK_END) buf_len = buf.tell() buf.seek(start) left = buf_len - start if not left: # only read from the wire if our read buffer is exhausted data = self._recv(self._rbufsize) if len(data) == size: # shortcut: skip the buffer if we read exactly size bytes return data buf = BytesIO() buf.write(data) buf.seek(0) del data # explicit free self._rbuf = buf return buf.read(size) def extract_capabilities(text): """Extract a capabilities list from a string, if present. :param text: String to extract from :return: Tuple with text with capabilities removed and list of capabilities """ if b"\0" not in text: return text, [] text, capabilities = text.rstrip().split(b"\0") return (text, capabilities.strip().split(b" ")) def extract_want_line_capabilities(text): """Extract a capabilities list from a want line, if present. Note that want lines have capabilities separated from the rest of the line by a space instead of a null byte. Thus want lines have the form: want obj-id cap1 cap2 ... :param text: Want line to extract from :return: Tuple with text with capabilities removed and list of capabilities """ split_text = text.rstrip().split(b" ") if len(split_text) < 3: return text, [] return (b" ".join(split_text[:2]), split_text[2:]) def ack_type(capabilities): """Extract the ack type from a capabilities list.""" if b'multi_ack_detailed' in capabilities: return MULTI_ACK_DETAILED elif b'multi_ack' in capabilities: return MULTI_ACK return SINGLE_ACK class BufferedPktLineWriter(object): """Writer that wraps its data in pkt-lines and has an independent buffer. Consecutive calls to write() wrap the data in a pkt-line and then buffers it until enough lines have been written such that their total length (including length prefix) reach the buffer size. """ def __init__(self, write, bufsize=65515): """Initialize the BufferedPktLineWriter. :param write: A write callback for the underlying writer. :param bufsize: The internal buffer size, including length prefixes. """ self._write = write self._bufsize = bufsize self._wbuf = BytesIO() self._buflen = 0 def write(self, data): """Write data, wrapping it in a pkt-line.""" line = pkt_line(data) line_len = len(line) over = self._buflen + line_len - self._bufsize if over >= 0: start = line_len - over self._wbuf.write(line[:start]) self.flush() else: start = 0 saved = line[start:] self._wbuf.write(saved) self._buflen += len(saved) def flush(self): """Flush all data from the buffer.""" data = self._wbuf.getvalue() if data: self._write(data) self._len = 0 self._wbuf = BytesIO() class PktLineParser(object): """Packet line parser that hands completed packets off to a callback. """ def __init__(self, handle_pkt): self.handle_pkt = handle_pkt self._readahead = BytesIO() def parse(self, data): """Parse a fragment of data and call back for any completed packets. """ self._readahead.write(data) buf = self._readahead.getvalue() if len(buf) < 4: return while len(buf) >= 4: size = int(buf[:4], 16) if size == 0: self.handle_pkt(None) buf = buf[4:] elif size <= len(buf): self.handle_pkt(buf[4:size]) buf = buf[size:] else: break self._readahead = BytesIO() self._readahead.write(buf) def get_tail(self): """Read back any unused data.""" return self._readahead.getvalue() diff --git a/dulwich/tests/test_server.py b/dulwich/tests/test_server.py index f3d957ec..9dcd3e83 100644 --- a/dulwich/tests/test_server.py +++ b/dulwich/tests/test_server.py @@ -1,1112 +1,1112 @@ # test_server.py -- Tests for the git server # Copyright (C) 2010 Google, Inc. # # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU # General Public License as public by the Free Software Foundation; version 2.0 # or (at your option) any later version. You can redistribute it and/or # modify it under the terms of either of these two licenses. # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # You should have received a copy of the licenses; if not, see # for a copy of the GNU General Public License # and for a copy of the Apache # License, Version 2.0. # """Tests for the smart protocol server.""" from io import BytesIO import os import shutil import tempfile import sys from dulwich.errors import ( GitProtocolError, NotGitRepository, UnexpectedCommandError, HangupException, ) from dulwich.object_store import ( MemoryObjectStore, ) from dulwich.repo import ( MemoryRepo, Repo, ) from dulwich.server import ( Backend, DictBackend, FileSystemBackend, MultiAckGraphWalkerImpl, MultiAckDetailedGraphWalkerImpl, PackHandler, _split_proto_line, serve_command, _find_shallow, _ProtocolGraphWalker, ReceivePackHandler, SingleAckGraphWalkerImpl, UploadPackHandler, update_server_info, ) from dulwich.tests import TestCase from dulwich.tests.utils import ( make_commit, make_tag, ) from dulwich.protocol import ( ZERO_SHA, ) ONE = b'1' * 40 TWO = b'2' * 40 THREE = b'3' * 40 FOUR = b'4' * 40 FIVE = b'5' * 40 SIX = b'6' * 40 class TestProto(object): def __init__(self): self._output = [] self._received = {0: [], 1: [], 2: [], 3: []} def set_output(self, output_lines): self._output = output_lines def read_pkt_line(self): if self._output: data = self._output.pop(0) if data is not None: return data.rstrip() + b'\n' else: # flush-pkt ('0000'). return None else: raise HangupException() def write_sideband(self, band, data): self._received[band].append(data) def write_pkt_line(self, data): self._received[0].append(data) def get_received_line(self, band=0): lines = self._received[band] return lines.pop(0) class TestGenericPackHandler(PackHandler): def __init__(self): PackHandler.__init__(self, Backend(), None) @classmethod def capabilities(cls): return [b'cap1', b'cap2', b'cap3'] @classmethod def required_capabilities(cls): return [b'cap2'] class HandlerTestCase(TestCase): def setUp(self): super(HandlerTestCase, self).setUp() self._handler = TestGenericPackHandler() def assertSucceeds(self, func, *args, **kwargs): try: func(*args, **kwargs) except GitProtocolError as e: self.fail(e) def test_capability_line(self): self.assertEqual( b' cap1 cap2 cap3', - self._handler.capability_line(['cap1', 'cap2', 'cap3'])) + self._handler.capability_line([b'cap1', b'cap2', b'cap3'])) def test_set_client_capabilities(self): set_caps = self._handler.set_client_capabilities self.assertSucceeds(set_caps, [b'cap2']) self.assertSucceeds(set_caps, [b'cap1', b'cap2']) # different order self.assertSucceeds(set_caps, [b'cap3', b'cap1', b'cap2']) # error cases self.assertRaises(GitProtocolError, set_caps, [b'capxxx', b'cap2']) self.assertRaises(GitProtocolError, set_caps, [b'cap1', b'cap3']) # ignore innocuous but unknown capabilities self.assertRaises(GitProtocolError, set_caps, [b'cap2', b'ignoreme']) self.assertFalse(b'ignoreme' in self._handler.capabilities()) self._handler.innocuous_capabilities = lambda: (b'ignoreme',) self.assertSucceeds(set_caps, [b'cap2', b'ignoreme']) def test_has_capability(self): self.assertRaises(GitProtocolError, self._handler.has_capability, b'cap') caps = self._handler.capabilities() self._handler.set_client_capabilities(caps) for cap in caps: self.assertTrue(self._handler.has_capability(cap)) self.assertFalse(self._handler.has_capability(b'capxxx')) class UploadPackHandlerTestCase(TestCase): def setUp(self): super(UploadPackHandlerTestCase, self).setUp() self._repo = MemoryRepo.init_bare([], {}) backend = DictBackend({b'/': self._repo}) self._handler = UploadPackHandler( backend, [b'/', b'host=lolcathost'], TestProto()) def test_progress(self): caps = self._handler.required_capabilities() self._handler.set_client_capabilities(caps) self._handler.progress(b'first message') self._handler.progress(b'second message') self.assertEqual(b'first message', self._handler.proto.get_received_line(2)) self.assertEqual(b'second message', self._handler.proto.get_received_line(2)) self.assertRaises(IndexError, self._handler.proto.get_received_line, 2) def test_no_progress(self): caps = list(self._handler.required_capabilities()) + [b'no-progress'] self._handler.set_client_capabilities(caps) self._handler.progress(b'first message') self._handler.progress(b'second message') self.assertRaises(IndexError, self._handler.proto.get_received_line, 2) def test_get_tagged(self): refs = { b'refs/tags/tag1': ONE, b'refs/tags/tag2': TWO, b'refs/heads/master': FOUR, # not a tag, no peeled value } # repo needs to peel this object self._repo.object_store.add_object(make_commit(id=FOUR)) self._repo.refs._update(refs) peeled = { b'refs/tags/tag1': b'1234' * 10, b'refs/tags/tag2': b'5678' * 10, } self._repo.refs._update_peeled(peeled) caps = list(self._handler.required_capabilities()) + [b'include-tag'] self._handler.set_client_capabilities(caps) self.assertEqual({b'1234' * 10: ONE, b'5678' * 10: TWO}, self._handler.get_tagged(refs, repo=self._repo)) # non-include-tag case caps = self._handler.required_capabilities() self._handler.set_client_capabilities(caps) self.assertEqual({}, self._handler.get_tagged(refs, repo=self._repo)) class FindShallowTests(TestCase): def setUp(self): super(FindShallowTests, self).setUp() self._store = MemoryObjectStore() def make_commit(self, **attrs): commit = make_commit(**attrs) self._store.add_object(commit) return commit def make_linear_commits(self, n, message=b''): commits = [] parents = [] for _ in range(n): commits.append(self.make_commit(parents=parents, message=message)) parents = [commits[-1].id] return commits def assertSameElements(self, expected, actual): self.assertEqual(set(expected), set(actual)) def test_linear(self): c1, c2, c3 = self.make_linear_commits(3) self.assertEqual((set([c3.id]), set([])), _find_shallow(self._store, [c3.id], 1)) self.assertEqual((set([c2.id]), set([c3.id])), _find_shallow(self._store, [c3.id], 2)) self.assertEqual((set([c1.id]), set([c2.id, c3.id])), _find_shallow(self._store, [c3.id], 3)) self.assertEqual((set([]), set([c1.id, c2.id, c3.id])), _find_shallow(self._store, [c3.id], 4)) def test_multiple_independent(self): a = self.make_linear_commits(2, message=b'a') b = self.make_linear_commits(2, message=b'b') c = self.make_linear_commits(2, message=b'c') heads = [a[1].id, b[1].id, c[1].id] self.assertEqual((set([a[0].id, b[0].id, c[0].id]), set(heads)), _find_shallow(self._store, heads, 2)) def test_multiple_overlapping(self): # Create the following commit tree: # 1--2 # \ # 3--4 c1, c2 = self.make_linear_commits(2) c3 = self.make_commit(parents=[c1.id]) c4 = self.make_commit(parents=[c3.id]) # 1 is shallow along the path from 4, but not along the path from 2. self.assertEqual((set([c1.id]), set([c1.id, c2.id, c3.id, c4.id])), _find_shallow(self._store, [c2.id, c4.id], 3)) def test_merge(self): c1 = self.make_commit() c2 = self.make_commit() c3 = self.make_commit(parents=[c1.id, c2.id]) self.assertEqual((set([c1.id, c2.id]), set([c3.id])), _find_shallow(self._store, [c3.id], 2)) def test_tag(self): c1, c2 = self.make_linear_commits(2) tag = make_tag(c2, name=b'tag') self._store.add_object(tag) self.assertEqual((set([c1.id]), set([c2.id])), _find_shallow(self._store, [tag.id], 2)) class TestUploadPackHandler(UploadPackHandler): @classmethod def required_capabilities(self): return [] class ReceivePackHandlerTestCase(TestCase): def setUp(self): super(ReceivePackHandlerTestCase, self).setUp() self._repo = MemoryRepo.init_bare([], {}) backend = DictBackend({b'/': self._repo}) self._handler = ReceivePackHandler( backend, [b'/', b'host=lolcathost'], TestProto()) def test_apply_pack_del_ref(self): refs = { b'refs/heads/master': TWO, b'refs/heads/fake-branch': ONE} self._repo.refs._update(refs) update_refs = [[ONE, ZERO_SHA, b'refs/heads/fake-branch'], ] self._handler.set_client_capabilities([b'delete-refs']) status = self._handler._apply_pack(update_refs) self.assertEqual(status[0][0], b'unpack') self.assertEqual(status[0][1], b'ok') self.assertEqual(status[1][0], b'refs/heads/fake-branch') self.assertEqual(status[1][1], b'ok') class ProtocolGraphWalkerEmptyTestCase(TestCase): def setUp(self): super(ProtocolGraphWalkerEmptyTestCase, self).setUp() self._repo = MemoryRepo.init_bare([], {}) backend = DictBackend({b'/': self._repo}) self._walker = _ProtocolGraphWalker( TestUploadPackHandler(backend, [b'/', b'host=lolcats'], TestProto()), self._repo.object_store, self._repo.get_peeled, self._repo.refs.get_symrefs) def test_empty_repository(self): # The server should wait for a flush packet. self._walker.proto.set_output([]) self.assertRaises(HangupException, self._walker.determine_wants, {}) self.assertEqual(None, self._walker.proto.get_received_line()) self._walker.proto.set_output([None]) self.assertEqual([], self._walker.determine_wants({})) self.assertEqual(None, self._walker.proto.get_received_line()) class ProtocolGraphWalkerTestCase(TestCase): def setUp(self): super(ProtocolGraphWalkerTestCase, self).setUp() # Create the following commit tree: # 3---5 # / # 1---2---4 commits = [ make_commit(id=ONE, parents=[], commit_time=111), make_commit(id=TWO, parents=[ONE], commit_time=222), make_commit(id=THREE, parents=[ONE], commit_time=333), make_commit(id=FOUR, parents=[TWO], commit_time=444), make_commit(id=FIVE, parents=[THREE], commit_time=555), ] self._repo = MemoryRepo.init_bare(commits, {}) backend = DictBackend({b'/': self._repo}) self._walker = _ProtocolGraphWalker( TestUploadPackHandler(backend, [b'/', b'host=lolcats'], TestProto()), self._repo.object_store, self._repo.get_peeled, self._repo.refs.get_symrefs) def test_all_wants_satisfied_no_haves(self): self._walker.set_wants([ONE]) self.assertFalse(self._walker.all_wants_satisfied([])) self._walker.set_wants([TWO]) self.assertFalse(self._walker.all_wants_satisfied([])) self._walker.set_wants([THREE]) self.assertFalse(self._walker.all_wants_satisfied([])) def test_all_wants_satisfied_have_root(self): self._walker.set_wants([ONE]) self.assertTrue(self._walker.all_wants_satisfied([ONE])) self._walker.set_wants([TWO]) self.assertTrue(self._walker.all_wants_satisfied([ONE])) self._walker.set_wants([THREE]) self.assertTrue(self._walker.all_wants_satisfied([ONE])) def test_all_wants_satisfied_have_branch(self): self._walker.set_wants([TWO]) self.assertTrue(self._walker.all_wants_satisfied([TWO])) # wrong branch self._walker.set_wants([THREE]) self.assertFalse(self._walker.all_wants_satisfied([TWO])) def test_all_wants_satisfied(self): self._walker.set_wants([FOUR, FIVE]) # trivial case: wants == haves self.assertTrue(self._walker.all_wants_satisfied([FOUR, FIVE])) # cases that require walking the commit tree self.assertTrue(self._walker.all_wants_satisfied([ONE])) self.assertFalse(self._walker.all_wants_satisfied([TWO])) self.assertFalse(self._walker.all_wants_satisfied([THREE])) self.assertTrue(self._walker.all_wants_satisfied([TWO, THREE])) def test_split_proto_line(self): allowed = (b'want', b'done', None) self.assertEqual((b'want', ONE), _split_proto_line(b'want ' + ONE + b'\n', allowed)) self.assertEqual((b'want', TWO), _split_proto_line(b'want ' + TWO + b'\n', allowed)) self.assertRaises(GitProtocolError, _split_proto_line, b'want xxxx\n', allowed) self.assertRaises(UnexpectedCommandError, _split_proto_line, b'have ' + THREE + b'\n', allowed) self.assertRaises(GitProtocolError, _split_proto_line, b'foo ' + FOUR + b'\n', allowed) self.assertRaises(GitProtocolError, _split_proto_line, b'bar', allowed) self.assertEqual((b'done', None), _split_proto_line(b'done\n', allowed)) self.assertEqual((None, None), _split_proto_line(b'', allowed)) def test_determine_wants(self): self._walker.proto.set_output([None]) self.assertEqual([], self._walker.determine_wants({})) self.assertEqual(None, self._walker.proto.get_received_line()) self._walker.proto.set_output([ b'want ' + ONE + b' multi_ack', b'want ' + TWO, None, ]) heads = { b'refs/heads/ref1': ONE, b'refs/heads/ref2': TWO, b'refs/heads/ref3': THREE, } self._repo.refs._update(heads) self.assertEqual([ONE, TWO], self._walker.determine_wants(heads)) self._walker.advertise_refs = True self.assertEqual([], self._walker.determine_wants(heads)) self._walker.advertise_refs = False self._walker.proto.set_output([b'want ' + FOUR + b' multi_ack', None]) self.assertRaises(GitProtocolError, self._walker.determine_wants, heads) self._walker.proto.set_output([None]) self.assertEqual([], self._walker.determine_wants(heads)) self._walker.proto.set_output( [b'want ' + ONE + b' multi_ack', b'foo', None]) self.assertRaises(GitProtocolError, self._walker.determine_wants, heads) self._walker.proto.set_output([b'want ' + FOUR + b' multi_ack', None]) self.assertRaises(GitProtocolError, self._walker.determine_wants, heads) def test_determine_wants_advertisement(self): self._walker.proto.set_output([None]) # advertise branch tips plus tag heads = { b'refs/heads/ref4': FOUR, b'refs/heads/ref5': FIVE, b'refs/heads/tag6': SIX, } self._repo.refs._update(heads) self._repo.refs._update_peeled(heads) self._repo.refs._update_peeled({b'refs/heads/tag6': FIVE}) self._walker.determine_wants(heads) lines = [] while True: line = self._walker.proto.get_received_line() if line is None: break # strip capabilities list if present if b'\x00' in line: line = line[:line.index(b'\x00')] lines.append(line.rstrip()) self.assertEqual([ FOUR + b' refs/heads/ref4', FIVE + b' refs/heads/ref5', FIVE + b' refs/heads/tag6^{}', SIX + b' refs/heads/tag6', ], sorted(lines)) # ensure peeled tag was advertised immediately following tag for i, line in enumerate(lines): if line.endswith(b' refs/heads/tag6'): self.assertEqual(FIVE + b' refs/heads/tag6^{}', lines[i+1]) # TODO: test commit time cutoff def _handle_shallow_request(self, lines, heads): self._walker.proto.set_output(lines + [None]) self._walker._handle_shallow_request(heads) def assertReceived(self, expected): self.assertEqual( expected, list(iter(self._walker.proto.get_received_line, None))) def test_handle_shallow_request_no_client_shallows(self): self._handle_shallow_request([b'deepen 2\n'], [FOUR, FIVE]) self.assertEqual(set([TWO, THREE]), self._walker.shallow) self.assertReceived([ b'shallow ' + TWO, b'shallow ' + THREE, ]) def test_handle_shallow_request_no_new_shallows(self): lines = [ b'shallow ' + TWO + b'\n', b'shallow ' + THREE + b'\n', b'deepen 2\n', ] self._handle_shallow_request(lines, [FOUR, FIVE]) self.assertEqual(set([TWO, THREE]), self._walker.shallow) self.assertReceived([]) def test_handle_shallow_request_unshallows(self): lines = [ b'shallow ' + TWO + b'\n', b'deepen 3\n', ] self._handle_shallow_request(lines, [FOUR, FIVE]) self.assertEqual(set([ONE]), self._walker.shallow) self.assertReceived([ b'shallow ' + ONE, b'unshallow ' + TWO, # THREE is unshallow but was is not shallow in the client ]) class TestProtocolGraphWalker(object): def __init__(self): self.acks = [] self.lines = [] self.wants_satisified = False self.http_req = None self.advertise_refs = False self._impl = None self.done_required = True self.done_received = False self._empty = False self.pack_sent = False def read_proto_line(self, allowed): command, sha = self.lines.pop(0) if allowed is not None: assert command in allowed return command, sha def send_ack(self, sha, ack_type=b''): self.acks.append((sha, ack_type)) def send_nak(self): self.acks.append((None, b'nak')) def all_wants_satisfied(self, haves): if haves: return self.wants_satisified def pop_ack(self): if not self.acks: return None return self.acks.pop(0) def handle_done(self): if not self._impl: return # Whether or not PACK is sent after is determined by this, so # record this value. self.pack_sent = self._impl.handle_done( self.done_required, self.done_received) return self.pack_sent def notify_done(self): self.done_received = True class AckGraphWalkerImplTestCase(TestCase): """Base setup and asserts for AckGraphWalker tests.""" def setUp(self): super(AckGraphWalkerImplTestCase, self).setUp() self._walker = TestProtocolGraphWalker() self._walker.lines = [ (b'have', TWO), (b'have', ONE), (b'have', THREE), (b'done', None), ] self._impl = self.impl_cls(self._walker) self._walker._impl = self._impl def assertNoAck(self): self.assertEqual(None, self._walker.pop_ack()) def assertAcks(self, acks): for sha, ack_type in acks: self.assertEqual((sha, ack_type), self._walker.pop_ack()) self.assertNoAck() def assertAck(self, sha, ack_type=b''): self.assertAcks([(sha, ack_type)]) def assertNak(self): self.assertAck(None, b'nak') def assertNextEquals(self, sha): self.assertEqual(sha, next(self._impl)) def assertNextEmpty(self): # This is necessary because of no-done - the assumption that it # it safe to immediately send out the final ACK is no longer # true but the test is still needed for it. TestProtocolWalker # does implement the handle_done which will determine whether # the final confirmation can be sent. self.assertRaises(IndexError, next, self._impl) self._walker.handle_done() class SingleAckGraphWalkerImplTestCase(AckGraphWalkerImplTestCase): impl_cls = SingleAckGraphWalkerImpl def test_single_ack(self): self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self._impl.ack(ONE) self.assertAck(ONE) self.assertNextEquals(THREE) self._impl.ack(THREE) self.assertNoAck() self.assertNextEquals(None) self.assertNoAck() def test_single_ack_flush(self): # same as ack test but ends with a flush-pkt instead of done self._walker.lines[-1] = (None, None) self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self._impl.ack(ONE) self.assertAck(ONE) self.assertNextEquals(THREE) self.assertNoAck() self.assertNextEquals(None) self.assertNoAck() def test_single_ack_nak(self): self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self.assertNoAck() self.assertNextEquals(THREE) self.assertNoAck() self.assertNextEquals(None) self.assertNextEmpty() self.assertNak() def test_single_ack_nak_flush(self): # same as nak test but ends with a flush-pkt instead of done self._walker.lines[-1] = (None, None) self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self.assertNoAck() self.assertNextEquals(THREE) self.assertNoAck() self.assertNextEquals(None) self.assertNextEmpty() self.assertNak() class MultiAckGraphWalkerImplTestCase(AckGraphWalkerImplTestCase): impl_cls = MultiAckGraphWalkerImpl def test_multi_ack(self): self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self._impl.ack(ONE) self.assertAck(ONE, b'continue') self.assertNextEquals(THREE) self._impl.ack(THREE) self.assertAck(THREE, b'continue') self.assertNextEquals(None) self.assertNextEmpty() self.assertAck(THREE) def test_multi_ack_partial(self): self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self._impl.ack(ONE) self.assertAck(ONE, b'continue') self.assertNextEquals(THREE) self.assertNoAck() self.assertNextEquals(None) self.assertNextEmpty() self.assertAck(ONE) def test_multi_ack_flush(self): self._walker.lines = [ (b'have', TWO), (None, None), (b'have', ONE), (b'have', THREE), (b'done', None), ] self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self.assertNak() # nak the flush-pkt self._impl.ack(ONE) self.assertAck(ONE, b'continue') self.assertNextEquals(THREE) self._impl.ack(THREE) self.assertAck(THREE, b'continue') self.assertNextEquals(None) self.assertNextEmpty() self.assertAck(THREE) def test_multi_ack_nak(self): self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self.assertNoAck() self.assertNextEquals(THREE) self.assertNoAck() self.assertNextEquals(None) self.assertNextEmpty() self.assertNak() class MultiAckDetailedGraphWalkerImplTestCase(AckGraphWalkerImplTestCase): impl_cls = MultiAckDetailedGraphWalkerImpl def test_multi_ack(self): self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self._impl.ack(ONE) self.assertAck(ONE, b'common') self.assertNextEquals(THREE) self._impl.ack(THREE) self.assertAck(THREE, b'common') # done is read. self._walker.wants_satisified = True self.assertNextEquals(None) self._walker.lines.append((None, None)) self.assertNextEmpty() self.assertAcks([(THREE, b'ready'), (None, b'nak'), (THREE, b'')]) # PACK is sent self.assertTrue(self._walker.pack_sent) def test_multi_ack_nodone(self): self._walker.done_required = False self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self._impl.ack(ONE) self.assertAck(ONE, b'common') self.assertNextEquals(THREE) self._impl.ack(THREE) self.assertAck(THREE, b'common') # done is read. self._walker.wants_satisified = True self.assertNextEquals(None) self._walker.lines.append((None, None)) self.assertNextEmpty() self.assertAcks([(THREE, b'ready'), (None, b'nak'), (THREE, b'')]) # PACK is sent self.assertTrue(self._walker.pack_sent) def test_multi_ack_flush_end(self): # transmission ends with a flush-pkt without a done but no-done is # assumed. self._walker.lines[-1] = (None, None) self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self._impl.ack(ONE) self.assertAck(ONE, b'common') self.assertNextEquals(THREE) self._impl.ack(THREE) self.assertAck(THREE, b'common') # no done is read self._walker.wants_satisified = True self.assertNextEmpty() self.assertAcks([(THREE, b'ready'), (None, b'nak')]) # PACK is NOT sent self.assertFalse(self._walker.pack_sent) def test_multi_ack_flush_end_nodone(self): # transmission ends with a flush-pkt without a done but no-done is # assumed. self._walker.lines[-1] = (None, None) self._walker.done_required = False self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self._impl.ack(ONE) self.assertAck(ONE, b'common') self.assertNextEquals(THREE) self._impl.ack(THREE) self.assertAck(THREE, b'common') # no done is read, but pretend it is (last 'ACK 'commit_id' '') self._walker.wants_satisified = True self.assertNextEmpty() self.assertAcks([(THREE, b'ready'), (None, b'nak'), (THREE, b'')]) # PACK is sent self.assertTrue(self._walker.pack_sent) def test_multi_ack_partial(self): self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self._impl.ack(ONE) self.assertAck(ONE, b'common') self.assertNextEquals(THREE) self.assertNoAck() self.assertNextEquals(None) self.assertNextEmpty() self.assertAck(ONE) def test_multi_ack_flush(self): # same as ack test but contains a flush-pkt in the middle self._walker.lines = [ (b'have', TWO), (None, None), (b'have', ONE), (b'have', THREE), (b'done', None), (None, None), ] self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self.assertNak() # nak the flush-pkt self._impl.ack(ONE) self.assertAck(ONE, b'common') self.assertNextEquals(THREE) self._impl.ack(THREE) self.assertAck(THREE, b'common') self._walker.wants_satisified = True self.assertNextEquals(None) self.assertNextEmpty() self.assertAcks([(THREE, b'ready'), (None, b'nak'), (THREE, b'')]) def test_multi_ack_nak(self): self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self.assertNoAck() self.assertNextEquals(THREE) self.assertNoAck() # Done is sent here. self.assertNextEquals(None) self.assertNextEmpty() self.assertNak() self.assertNextEmpty() self.assertTrue(self._walker.pack_sent) def test_multi_ack_nak_nodone(self): self._walker.done_required = False self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self.assertNoAck() self.assertNextEquals(THREE) self.assertNoAck() # Done is sent here. self.assertFalse(self._walker.pack_sent) self.assertNextEquals(None) self.assertNextEmpty() self.assertTrue(self._walker.pack_sent) self.assertNak() self.assertNextEmpty() def test_multi_ack_nak_flush(self): # same as nak test but contains a flush-pkt in the middle self._walker.lines = [ (b'have', TWO), (None, None), (b'have', ONE), (b'have', THREE), (b'done', None), ] self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self.assertNak() self.assertNextEquals(THREE) self.assertNoAck() self.assertNextEquals(None) self.assertNextEmpty() self.assertNak() def test_multi_ack_stateless(self): # transmission ends with a flush-pkt self._walker.lines[-1] = (None, None) self._walker.http_req = True self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self.assertNoAck() self.assertNextEquals(THREE) self.assertNoAck() self.assertFalse(self._walker.pack_sent) self.assertNextEquals(None) self.assertNak() self.assertNextEmpty() self.assertNoAck() self.assertFalse(self._walker.pack_sent) def test_multi_ack_stateless_nodone(self): self._walker.done_required = False # transmission ends with a flush-pkt self._walker.lines[-1] = (None, None) self._walker.http_req = True self.assertNextEquals(TWO) self.assertNoAck() self.assertNextEquals(ONE) self.assertNoAck() self.assertNextEquals(THREE) self.assertNoAck() self.assertFalse(self._walker.pack_sent) self.assertNextEquals(None) self.assertNak() self.assertNextEmpty() self.assertNoAck() # PACK will still not be sent. self.assertFalse(self._walker.pack_sent) class FileSystemBackendTests(TestCase): """Tests for FileSystemBackend.""" def setUp(self): super(FileSystemBackendTests, self).setUp() self.path = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.path) self.repo = Repo.init(self.path) if sys.platform == 'win32': self.backend = FileSystemBackend(self.path[0] + ':' + os.sep) else: self.backend = FileSystemBackend() def test_nonexistant(self): self.assertRaises(NotGitRepository, self.backend.open_repository, "/does/not/exist/unless/foo") def test_absolute(self): repo = self.backend.open_repository(self.path) self.assertEqual( os.path.normcase(os.path.abspath(repo.path)), os.path.normcase(os.path.abspath(self.repo.path))) def test_child(self): self.assertRaises( NotGitRepository, self.backend.open_repository, os.path.join(self.path, "foo")) def test_bad_repo_path(self): backend = FileSystemBackend() self.assertRaises(NotGitRepository, lambda: backend.open_repository('/ups')) class DictBackendTests(TestCase): """Tests for DictBackend.""" def test_nonexistant(self): repo = MemoryRepo.init_bare([], {}) backend = DictBackend({b'/': repo}) self.assertRaises( NotGitRepository, backend.open_repository, "/does/not/exist/unless/foo") def test_bad_repo_path(self): repo = MemoryRepo.init_bare([], {}) backend = DictBackend({b'/': repo}) self.assertRaises(NotGitRepository, lambda: backend.open_repository('/ups')) class ServeCommandTests(TestCase): """Tests for serve_command.""" def setUp(self): super(ServeCommandTests, self).setUp() self.backend = DictBackend({}) def serve_command(self, handler_cls, args, inf, outf): return serve_command( handler_cls, [b"test"] + args, backend=self.backend, inf=inf, outf=outf) def test_receive_pack(self): commit = make_commit(id=ONE, parents=[], commit_time=111) self.backend.repos[b"/"] = MemoryRepo.init_bare( [commit], {b"refs/heads/master": commit.id}) outf = BytesIO() exitcode = self.serve_command(ReceivePackHandler, [b"/"], BytesIO(b"0000"), outf) outlines = outf.getvalue().splitlines() self.assertEqual(2, len(outlines)) self.assertEqual( b"1111111111111111111111111111111111111111 refs/heads/master", outlines[0][4:].split(b"\x00")[0]) self.assertEqual(b"0000", outlines[-1]) self.assertEqual(0, exitcode) class UpdateServerInfoTests(TestCase): """Tests for update_server_info.""" def setUp(self): super(UpdateServerInfoTests, self).setUp() self.path = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.path) self.repo = Repo.init(self.path) def test_empty(self): update_server_info(self.repo) with open(os.path.join(self.path, ".git", "info", "refs"), 'rb') as f: self.assertEqual(b'', f.read()) p = os.path.join(self.path, ".git", "objects", "info", "packs") with open(p, 'rb') as f: self.assertEqual(b'', f.read()) def test_simple(self): commit_id = self.repo.do_commit( message=b"foo", committer=b"Joe Example ", ref=b"refs/heads/foo") update_server_info(self.repo) with open(os.path.join(self.path, ".git", "info", "refs"), 'rb') as f: self.assertEqual(f.read(), commit_id + b'\trefs/heads/foo\n') p = os.path.join(self.path, ".git", "objects", "info", "packs") with open(p, 'rb') as f: self.assertEqual(f.read(), b'')