diff --git a/bin/dulwich b/bin/dulwich index 1626b56b..5b858e01 100755 --- a/bin/dulwich +++ b/bin/dulwich @@ -1,700 +1,705 @@ #!/usr/bin/python -u # # dulwich - Simple command-line interface to Dulwich # Copyright (C) 2008-2011 Jelmer Vernooij # vim: expandtab # # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU # General Public License as public by the Free Software Foundation; version 2.0 # or (at your option) any later version. You can redistribute it and/or # modify it under the terms of either of these two licenses. # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # You should have received a copy of the licenses; if not, see # for a copy of the GNU General Public License # and for a copy of the Apache # License, Version 2.0. # """Simple command-line interface to Dulwich> This is a very simple command-line wrapper for Dulwich. It is by no means intended to be a full-blown Git command-line interface but just a way to test Dulwich. """ import os import sys from getopt import getopt import optparse import signal def signal_int(signal, frame): sys.exit(1) def signal_quit(signal, frame): import pdb pdb.set_trace() if 'DULWICH_PDB' in os.environ: signal.signal(signal.SIGQUIT, signal_quit) signal.signal(signal.SIGINT, signal_int) from dulwich import porcelain from dulwich.client import get_transport_and_path from dulwich.errors import ApplyDeltaError from dulwich.index import Index from dulwich.pack import Pack, sha_to_hex from dulwich.patch import write_tree_diff from dulwich.repo import Repo class Command(object): """A Dulwich subcommand.""" def run(self, args): """Run the command.""" raise NotImplementedError(self.run) class cmd_archive(Command): def run(self, args): parser = optparse.OptionParser() parser.add_option("--remote", type=str, help="Retrieve archive from specified remote repo") options, args = parser.parse_args(args) committish = args.pop(0) if options.remote: client, path = get_transport_and_path(options.remote) client.archive(path, committish, sys.stdout.write, write_error=sys.stderr.write) else: porcelain.archive('.', committish, outstream=sys.stdout, errstream=sys.stderr) class cmd_add(Command): def run(self, args): opts, args = getopt(args, "", []) porcelain.add(".", paths=args) class cmd_rm(Command): def run(self, args): opts, args = getopt(args, "", []) porcelain.rm(".", paths=args) class cmd_fetch_pack(Command): def run(self, args): opts, args = getopt(args, "", ["all"]) opts = dict(opts) client, path = get_transport_and_path(args.pop(0)) r = Repo(".") if "--all" in opts: determine_wants = r.object_store.determine_wants_all else: determine_wants = lambda x: [y for y in args if not y in r.object_store] client.fetch(path, r, determine_wants) class cmd_fetch(Command): def run(self, args): opts, args = getopt(args, "", []) opts = dict(opts) client, path = get_transport_and_path(args.pop(0)) r = Repo(".") if "--all" in opts: determine_wants = r.object_store.determine_wants_all refs = client.fetch(path, r, progress=sys.stdout.write) print("Remote refs:") for item in refs.items(): print("%s -> %s" % item) class cmd_fsck(Command): def run(self, args): opts, args = getopt(args, "", []) opts = dict(opts) for (obj, msg) in porcelain.fsck('.'): print("%s: %s" % (obj, msg)) class cmd_log(Command): def run(self, args): parser = optparse.OptionParser() parser.add_option("--reverse", dest="reverse", action="store_true", help="Reverse order in which entries are printed") parser.add_option("--name-status", dest="name_status", action="store_true", help="Print name/status for each changed file") options, args = parser.parse_args(args) porcelain.log(".", paths=args, reverse=options.reverse, name_status=options.name_status, outstream=sys.stdout) class cmd_diff(Command): def run(self, args): opts, args = getopt(args, "", []) if args == []: print("Usage: dulwich diff COMMITID") sys.exit(1) r = Repo(".") commit_id = args[0] commit = r[commit_id] parent_commit = r[commit.parents[0]] write_tree_diff(sys.stdout, r.object_store, parent_commit.tree, commit.tree) class cmd_dump_pack(Command): def run(self, args): opts, args = getopt(args, "", []) if args == []: print("Usage: dulwich dump-pack FILENAME") sys.exit(1) basename, _ = os.path.splitext(args[0]) x = Pack(basename) print("Object names checksum: %s" % x.name()) print("Checksum: %s" % sha_to_hex(x.get_stored_checksum())) if not x.check(): print("CHECKSUM DOES NOT MATCH") print("Length: %d" % len(x)) for name in x: try: print("\t%s" % x[name]) except KeyError as k: print("\t%s: Unable to resolve base %s" % (name, k)) except ApplyDeltaError as e: print("\t%s: Unable to apply delta: %r" % (name, e)) class cmd_dump_index(Command): def run(self, args): opts, args = getopt(args, "", []) if args == []: print("Usage: dulwich dump-index FILENAME") sys.exit(1) filename = args[0] idx = Index(filename) for o in idx: print(o, idx[o]) class cmd_init(Command): def run(self, args): opts, args = getopt(args, "", ["bare"]) opts = dict(opts) if args == []: path = os.getcwd() else: path = args[0] porcelain.init(path, bare=("--bare" in opts)) class cmd_clone(Command): def run(self, args): - opts, args = getopt(args, "", ["bare"]) - opts = dict(opts) + parser = optparse.OptionParser() + parser.add_option("--bare", dest="bare", + help="Whether to create a bare repository.", + action="store_true") + parser.add_option("--depth", dest="depth", + type=int, help="Depth at which to fetch") + options, args = parser.parse_args(args) if args == []: print("usage: dulwich clone host:path [PATH]") sys.exit(1) source = args.pop(0) if len(args) > 0: target = args.pop(0) else: target = None - porcelain.clone(source, target, bare=("--bare" in opts)) + porcelain.clone(source, target, bare=options.bare, depth=options.depth) class cmd_commit(Command): def run(self, args): opts, args = getopt(args, "", ["message"]) opts = dict(opts) porcelain.commit(".", message=opts["--message"]) class cmd_commit_tree(Command): def run(self, args): opts, args = getopt(args, "", ["message"]) if args == []: print("usage: dulwich commit-tree tree") sys.exit(1) opts = dict(opts) porcelain.commit_tree(".", tree=args[0], message=opts["--message"]) class cmd_update_server_info(Command): def run(self, args): porcelain.update_server_info(".") class cmd_symbolic_ref(Command): def run(self, args): opts, args = getopt(args, "", ["ref-name", "force"]) if not args: print("Usage: dulwich symbolic-ref REF_NAME [--force]") sys.exit(1) ref_name = args.pop(0) porcelain.symbolic_ref(".", ref_name=ref_name, force='--force' in args) class cmd_show(Command): def run(self, args): opts, args = getopt(args, "", []) porcelain.show(".", args) class cmd_diff_tree(Command): def run(self, args): opts, args = getopt(args, "", []) if len(args) < 2: print("Usage: dulwich diff-tree OLD-TREE NEW-TREE") sys.exit(1) porcelain.diff_tree(".", args[0], args[1]) class cmd_rev_list(Command): def run(self, args): opts, args = getopt(args, "", []) if len(args) < 1: print('Usage: dulwich rev-list COMMITID...') sys.exit(1) porcelain.rev_list('.', args) class cmd_tag(Command): def run(self, args): opts, args = getopt(args, '', []) if len(args) < 2: print('Usage: dulwich tag NAME') sys.exit(1) porcelain.tag('.', args[0]) class cmd_repack(Command): def run(self, args): opts, args = getopt(args, "", []) opts = dict(opts) porcelain.repack('.') class cmd_reset(Command): def run(self, args): opts, args = getopt(args, "", ["hard", "soft", "mixed"]) opts = dict(opts) mode = "" if "--hard" in opts: mode = "hard" elif "--soft" in opts: mode = "soft" elif "--mixed" in opts: mode = "mixed" porcelain.reset('.', mode=mode, *args) class cmd_daemon(Command): def run(self, args): from dulwich import log_utils from dulwich.protocol import TCP_GIT_PORT parser = optparse.OptionParser() parser.add_option("-l", "--listen_address", dest="listen_address", default="localhost", help="Binding IP address.") parser.add_option("-p", "--port", dest="port", type=int, default=TCP_GIT_PORT, help="Binding TCP port.") options, args = parser.parse_args(args) log_utils.default_logging_config() if len(args) >= 1: gitdir = args[0] else: gitdir = '.' from dulwich import porcelain porcelain.daemon(gitdir, address=options.listen_address, port=options.port) class cmd_web_daemon(Command): def run(self, args): from dulwich import log_utils parser = optparse.OptionParser() parser.add_option("-l", "--listen_address", dest="listen_address", default="", help="Binding IP address.") parser.add_option("-p", "--port", dest="port", type=int, default=8000, help="Binding TCP port.") options, args = parser.parse_args(args) log_utils.default_logging_config() if len(args) >= 1: gitdir = args[0] else: gitdir = '.' from dulwich import porcelain porcelain.web_daemon(gitdir, address=options.listen_address, port=options.port) class cmd_receive_pack(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) if len(args) >= 1: gitdir = args[0] else: gitdir = '.' porcelain.receive_pack(gitdir) class cmd_upload_pack(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) if len(args) >= 1: gitdir = args[0] else: gitdir = '.' porcelain.upload_pack(gitdir) class cmd_status(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) if len(args) >= 1: gitdir = args[0] else: gitdir = '.' status = porcelain.status(gitdir) if any(names for (kind, names) in status.staged.items()): sys.stdout.write("Changes to be committed:\n\n") for kind, names in status.staged.items(): for name in names: sys.stdout.write("\t%s: %s\n" % ( kind, name.decode(sys.getfilesystemencoding()))) sys.stdout.write("\n") if status.unstaged: sys.stdout.write("Changes not staged for commit:\n\n") for name in status.unstaged: sys.stdout.write("\t%s\n" % name.decode(sys.getfilesystemencoding())) sys.stdout.write("\n") if status.untracked: sys.stdout.write("Untracked files:\n\n") for name in status.untracked: sys.stdout.write("\t%s\n" % name) sys.stdout.write("\n") class cmd_ls_remote(Command): def run(self, args): opts, args = getopt(args, '', []) if len(args) < 1: print('Usage: dulwich ls-remote URL') sys.exit(1) refs = porcelain.ls_remote(args[0]) for ref in sorted(refs): sys.stdout.write("%s\t%s\n" % (ref, refs[ref])) class cmd_ls_tree(Command): def run(self, args): parser = optparse.OptionParser() parser.add_option("-r", "--recursive", action="store_true", help="Recusively list tree contents.") parser.add_option("--name-only", action="store_true", help="Only display name.") options, args = parser.parse_args(args) try: treeish = args.pop(0) except IndexError: treeish = None porcelain.ls_tree( '.', treeish, outstream=sys.stdout, recursive=options.recursive, name_only=options.name_only) class cmd_pack_objects(Command): def run(self, args): opts, args = getopt(args, '', ['stdout']) opts = dict(opts) if len(args) < 1 and not '--stdout' in args: print('Usage: dulwich pack-objects basename') sys.exit(1) object_ids = [l.strip() for l in sys.stdin.readlines()] basename = args[0] if '--stdout' in opts: packf = getattr(sys.stdout, 'buffer', sys.stdout) idxf = None close = [] else: packf = open(basename + '.pack', 'w') idxf = open(basename + '.idx', 'w') close = [packf, idxf] porcelain.pack_objects('.', object_ids, packf, idxf) for f in close: f.close() class cmd_pull(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) try: from_location = args[0] except IndexError: from_location = None porcelain.pull('.', from_location) class cmd_push(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) if len(args) < 2: print("Usage: dulwich push TO-LOCATION REFSPEC..") sys.exit(1) to_location = args[0] refspecs = args[1:] porcelain.push('.', to_location, refspecs) class cmd_remote_add(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) porcelain.remote_add('.', args[0], args[1]) class SuperCommand(Command): subcommands = {} def run(self, args): if not args: print("Supported subcommands: %s" % ', '.join(self.subcommands.keys())) return False cmd = args[0] try: cmd_kls = self.subcommands[cmd] except KeyError: print('No such subcommand: %s' % args[0]) return False return cmd_kls().run(args[1:]) class cmd_remote(SuperCommand): subcommands = { "add": cmd_remote_add, } class cmd_check_ignore(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) ret = 1 for path in porcelain.check_ignore('.', args): print(path) ret = 0 return ret class cmd_check_mailmap(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) for arg in args: canonical_identity = porcelain.check_mailmap('.', arg) print(canonical_identity) class cmd_stash_list(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) for i, entry in porcelain.stash_list('.'): print("stash@{%d}: %s" % (i, entry.message.rstrip('\n'))) class cmd_stash_push(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) porcelain.stash_push('.') print("Saved working directory and index state") class cmd_stash_pop(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) porcelain.stash_pop('.') print("Restrored working directory and index state") class cmd_stash(SuperCommand): subcommands = { "list": cmd_stash_list, "pop": cmd_stash_pop, "push": cmd_stash_push, } class cmd_ls_files(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) for name in porcelain.ls_files('.'): print(name) class cmd_describe(Command): def run(self, args): parser = optparse.OptionParser() options, args = parser.parse_args(args) print(porcelain.describe('.')) class cmd_help(Command): def run(self, args): parser = optparse.OptionParser() parser.add_option("-a", "--all", dest="all", action="store_true", help="List all commands.") options, args = parser.parse_args(args) if options.all: print('Available commands:') for cmd in sorted(commands): print(' %s' % cmd) else: print("""\ The dulwich command line tool is currently a very basic frontend for the Dulwich python module. For full functionality, please see the API reference. For a list of supported commands, see 'dulwich help -a'. """) commands = { "add": cmd_add, "archive": cmd_archive, "check-ignore": cmd_check_ignore, "check-mailmap": cmd_check_mailmap, "clone": cmd_clone, "commit": cmd_commit, "commit-tree": cmd_commit_tree, "describe": cmd_describe, "daemon": cmd_daemon, "diff": cmd_diff, "diff-tree": cmd_diff_tree, "dump-pack": cmd_dump_pack, "dump-index": cmd_dump_index, "fetch-pack": cmd_fetch_pack, "fetch": cmd_fetch, "fsck": cmd_fsck, "help": cmd_help, "init": cmd_init, "log": cmd_log, "ls-files": cmd_ls_files, "ls-remote": cmd_ls_remote, "ls-tree": cmd_ls_tree, "pack-objects": cmd_pack_objects, "pull": cmd_pull, "push": cmd_push, "receive-pack": cmd_receive_pack, "remote": cmd_remote, "repack": cmd_repack, "reset": cmd_reset, "rev-list": cmd_rev_list, "rm": cmd_rm, "show": cmd_show, "stash": cmd_stash, "status": cmd_status, "symbolic-ref": cmd_symbolic_ref, "tag": cmd_tag, "update-server-info": cmd_update_server_info, "upload-pack": cmd_upload_pack, "web-daemon": cmd_web_daemon, } if len(sys.argv) < 2: print("Usage: %s <%s> [OPTIONS...]" % (sys.argv[0], "|".join(commands.keys()))) sys.exit(1) cmd = sys.argv[1] try: cmd_kls = commands[cmd] except KeyError: print("No such subcommand: %s" % cmd) sys.exit(1) # TODO(jelmer): Return non-0 on errors cmd_kls().run(sys.argv[2:]) diff --git a/dulwich/client.py b/dulwich/client.py index c938107c..b13427a6 100644 --- a/dulwich/client.py +++ b/dulwich/client.py @@ -1,1696 +1,1711 @@ # client.py -- Implementation of the client side git protocols # Copyright (C) 2008-2013 Jelmer Vernooij # # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU # General Public License as public by the Free Software Foundation; version 2.0 # or (at your option) any later version. You can redistribute it and/or # modify it under the terms of either of these two licenses. # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # You should have received a copy of the licenses; if not, see # for a copy of the GNU General Public License # and for a copy of the Apache # License, Version 2.0. # """Client side support for the Git protocol. The Dulwich client supports the following capabilities: * thin-pack * multi_ack_detailed * multi_ack * side-band-64k * ofs-delta * quiet * report-status * delete-refs Known capabilities that are not supported: * shallow * no-progress * include-tag """ from contextlib import closing from io import BytesIO, BufferedReader import select import socket import subprocess import sys try: from urllib import quote as urlquote from urllib import unquote as urlunquote except ImportError: from urllib.parse import quote as urlquote from urllib.parse import unquote as urlunquote try: import urlparse except ImportError: import urllib.parse as urlparse import dulwich from dulwich.errors import ( GitProtocolError, NotGitRepository, SendPackError, UpdateRefsError, ) from dulwich.protocol import ( HangupException, _RBUFSIZE, agent_string, capability_agent, extract_capability_names, CAPABILITY_AGENT, CAPABILITY_DELETE_REFS, CAPABILITY_MULTI_ACK, CAPABILITY_MULTI_ACK_DETAILED, CAPABILITY_OFS_DELTA, CAPABILITY_QUIET, CAPABILITY_REPORT_STATUS, + CAPABILITY_SHALLOW, CAPABILITY_SYMREF, CAPABILITY_SIDE_BAND_64K, CAPABILITY_THIN_PACK, CAPABILITIES_REF, KNOWN_RECEIVE_CAPABILITIES, KNOWN_UPLOAD_CAPABILITIES, + COMMAND_DEEPEN, COMMAND_DONE, COMMAND_HAVE, COMMAND_WANT, SIDE_BAND_CHANNEL_DATA, SIDE_BAND_CHANNEL_PROGRESS, SIDE_BAND_CHANNEL_FATAL, PktLineParser, Protocol, ProtocolFile, TCP_GIT_PORT, ZERO_SHA, extract_capabilities, parse_capability, ) from dulwich.pack import ( write_pack_data, write_pack_objects, ) from dulwich.refs import ( read_info_refs, ANNOTATED_TAG_SUFFIX, ) class InvalidWants(Exception): """Invalid wants.""" def __init__(self, wants): Exception.__init__( self, "requested wants not in server provided refs: %r" % wants) def _fileno_can_read(fileno): """Check if a file descriptor is readable.""" return len(select.select([fileno], [], [], 0)[0]) > 0 def _win32_peek_avail(handle): """Wrapper around PeekNamedPipe to check how many bytes are available.""" from ctypes import byref, wintypes, windll c_avail = wintypes.DWORD() c_message = wintypes.DWORD() success = windll.kernel32.PeekNamedPipe( handle, None, 0, None, byref(c_avail), byref(c_message)) if not success: raise OSError(wintypes.GetLastError()) return c_avail.value COMMON_CAPABILITIES = [CAPABILITY_OFS_DELTA, CAPABILITY_SIDE_BAND_64K] UPLOAD_CAPABILITIES = ([CAPABILITY_THIN_PACK, CAPABILITY_MULTI_ACK, - CAPABILITY_MULTI_ACK_DETAILED] + COMMON_CAPABILITIES) + CAPABILITY_MULTI_ACK_DETAILED, CAPABILITY_SHALLOW] + + COMMON_CAPABILITIES) RECEIVE_CAPABILITIES = [CAPABILITY_REPORT_STATUS] + COMMON_CAPABILITIES class ReportStatusParser(object): """Handle status as reported by servers with 'report-status' capability. """ def __init__(self): self._done = False self._pack_status = None self._ref_status_ok = True self._ref_statuses = [] def check(self): """Check if there were any errors and, if so, raise exceptions. :raise SendPackError: Raised when the server could not unpack :raise UpdateRefsError: Raised when refs could not be updated """ if self._pack_status not in (b'unpack ok', None): raise SendPackError(self._pack_status) if not self._ref_status_ok: ref_status = {} ok = set() for status in self._ref_statuses: if b' ' not in status: # malformed response, move on to the next one continue status, ref = status.split(b' ', 1) if status == b'ng': if b' ' in ref: ref, status = ref.split(b' ', 1) else: ok.add(ref) ref_status[ref] = status # TODO(jelmer): don't assume encoding of refs is ascii. raise UpdateRefsError(', '.join([ refname.decode('ascii') for refname in ref_status if refname not in ok]) + ' failed to update', ref_status=ref_status) def handle_packet(self, pkt): """Handle a packet. :raise GitProtocolError: Raised when packets are received after a flush packet. """ if self._done: raise GitProtocolError("received more data after status report") if pkt is None: self._done = True return if self._pack_status is None: self._pack_status = pkt.strip() else: ref_status = pkt.strip() self._ref_statuses.append(ref_status) if not ref_status.startswith(b'ok '): self._ref_status_ok = False def read_pkt_refs(proto): server_capabilities = None refs = {} # Receive refs from server for pkt in proto.read_pkt_seq(): (sha, ref) = pkt.rstrip(b'\n').split(None, 1) if sha == b'ERR': raise GitProtocolError(ref.decode('utf-8', 'replace')) if server_capabilities is None: (ref, server_capabilities) = extract_capabilities(ref) refs[ref] = sha if len(refs) == 0: return {}, set([]) if refs == {CAPABILITIES_REF: ZERO_SHA}: refs = {} return refs, set(server_capabilities) class FetchPackResult(object): """Result of a fetch-pack operation. :var refs: Dictionary with all remote refs :var symrefs: Dictionary with remote symrefs :var agent: User agent string """ _FORWARDED_ATTRS = [ 'clear', 'copy', 'fromkeys', 'get', 'has_key', 'items', 'iteritems', 'iterkeys', 'itervalues', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values', 'viewitems', 'viewkeys', 'viewvalues'] def __init__(self, refs, symrefs, agent): self.refs = refs self.symrefs = symrefs self.agent = agent def _warn_deprecated(self): import warnings warnings.warn( "Use FetchPackResult.refs instead.", DeprecationWarning, stacklevel=3) def __eq__(self, other): if isinstance(other, dict): self._warn_deprecated() return (self.refs == other) return (self.refs == other.refs and self.symrefs == other.symrefs and self.agent == other.agent) def __contains__(self, name): self._warn_deprecated() return name in self.refs def __getitem__(self, name): self._warn_deprecated() return self.refs[name] def __len__(self): self._warn_deprecated() return len(self.refs) def __iter__(self): self._warn_deprecated() return iter(self.refs) def __getattribute__(self, name): if name in type(self)._FORWARDED_ATTRS: self._warn_deprecated() return getattr(self.refs, name) return super(FetchPackResult, self).__getattribute__(name) def __repr__(self): return "%s(%r, %r, %r)" % ( self.__class__.__name__, self.refs, self.symrefs, self.agent) # TODO(durin42): this doesn't correctly degrade if the server doesn't # support some capabilities. This should work properly with servers # that don't support multi_ack. class GitClient(object): """Git smart server client. """ def __init__(self, thin_packs=True, report_activity=None, quiet=False): """Create a new GitClient instance. :param thin_packs: Whether or not thin packs should be retrieved :param report_activity: Optional callback for reporting transport activity. """ self._report_activity = report_activity self._report_status_parser = None self._fetch_capabilities = set(UPLOAD_CAPABILITIES) self._fetch_capabilities.add(capability_agent()) self._send_capabilities = set(RECEIVE_CAPABILITIES) self._send_capabilities.add(capability_agent()) if quiet: self._send_capabilities.add(CAPABILITY_QUIET) if not thin_packs: self._fetch_capabilities.remove(CAPABILITY_THIN_PACK) def get_url(self, path): """Retrieves full url to given path. :param path: Repository path (as string) :return: Url to path (as string) """ raise NotImplementedError(self.get_url) @classmethod def from_parsedurl(cls, parsedurl, **kwargs): """Create an instance of this client from a urlparse.parsed object. :param parsedurl: Result of urlparse.urlparse() :return: A `GitClient` object """ raise NotImplementedError(cls.from_parsedurl) def send_pack(self, path, update_refs, generate_pack_data, progress=None): """Upload a pack to a remote repository. :param path: Repository path (as bytestring) :param update_refs: Function to determine changes to remote refs. Receive dict with existing remote refs, returns dict with changed refs (name -> sha, where sha=ZERO_SHA for deletions) :param generate_pack_data: Function that can return a tuple with number of objects and list of pack data to include :param progress: Optional progress function :raises SendPackError: if server rejects the pack data :raises UpdateRefsError: if the server supports report-status and rejects ref updates :return: new_refs dictionary containing the changes that were made {refname: new_ref}, including deleted refs. """ raise NotImplementedError(self.send_pack) - def fetch(self, path, target, determine_wants=None, progress=None): + def fetch(self, path, target, determine_wants=None, progress=None, + depth=None): """Fetch into a target repository. :param path: Path to fetch from (as bytestring) :param target: Target repository to fetch into :param determine_wants: Optional function to determine what refs to fetch. Receives dictionary of name->sha, should return list of shas to fetch. Defaults to all shas. :param progress: Optional progress function + :param depth: Depth to fetch at :return: Dictionary with all remote refs (not just those fetched) """ if determine_wants is None: determine_wants = target.object_store.determine_wants_all if CAPABILITY_THIN_PACK in self._fetch_capabilities: # TODO(jelmer): Avoid reading entire file into memory and # only processing it after the whole file has been fetched. f = BytesIO() def commit(): if f.tell(): f.seek(0) target.object_store.add_thin_pack(f.read, None) def abort(): pass else: f, commit, abort = target.object_store.add_pack() try: result = self.fetch_pack( path, determine_wants, target.get_graph_walker(), f.write, - progress) + progress=progress, depth=depth) except BaseException: abort() raise else: commit() return result def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress=None): """Retrieve a pack from a git smart server. :param path: Remote path to fetch from :param determine_wants: Function determine what refs to fetch. Receives dictionary of name->sha, should return list of shas to fetch. :param graph_walker: Object with next() and ack(). :param pack_data: Callback called for each bit of data in the pack :param progress: Callback for progress reports (strings) :return: FetchPackResult object """ raise NotImplementedError(self.fetch_pack) def get_refs(self, path): """Retrieve the current refs from a git smart server. :param path: Path to the repo to fetch from. (as bytestring) """ raise NotImplementedError(self.get_refs) def _parse_status_report(self, proto): unpack = proto.read_pkt_line().strip() if unpack != b'unpack ok': st = True # flush remaining error data while st is not None: st = proto.read_pkt_line() raise SendPackError(unpack) statuses = [] errs = False ref_status = proto.read_pkt_line() while ref_status: ref_status = ref_status.strip() statuses.append(ref_status) if not ref_status.startswith(b'ok '): errs = True ref_status = proto.read_pkt_line() if errs: ref_status = {} ok = set() for status in statuses: if b' ' not in status: # malformed response, move on to the next one continue status, ref = status.split(b' ', 1) if status == b'ng': if b' ' in ref: ref, status = ref.split(b' ', 1) else: ok.add(ref) ref_status[ref] = status raise UpdateRefsError(', '.join([ refname for refname in ref_status if refname not in ok]) + b' failed to update', ref_status=ref_status) def _read_side_band64k_data(self, proto, channel_callbacks): """Read per-channel data. This requires the side-band-64k capability. :param proto: Protocol object to read from :param channel_callbacks: Dictionary mapping channels to packet handlers to use. None for a callback discards channel data. """ for pkt in proto.read_pkt_seq(): channel = ord(pkt[:1]) pkt = pkt[1:] try: cb = channel_callbacks[channel] except KeyError: raise AssertionError('Invalid sideband channel %d' % channel) else: if cb is not None: cb(pkt) def _handle_receive_pack_head(self, proto, capabilities, old_refs, new_refs): """Handle the head of a 'git-receive-pack' request. :param proto: Protocol object to read from :param capabilities: List of negotiated capabilities :param old_refs: Old refs, as received from the server :param new_refs: Refs to change :return: (have, want) tuple """ want = [] have = [x for x in old_refs.values() if not x == ZERO_SHA] sent_capabilities = False for refname in new_refs: if not isinstance(refname, bytes): raise TypeError('refname is not a bytestring: %r' % refname) old_sha1 = old_refs.get(refname, ZERO_SHA) if not isinstance(old_sha1, bytes): raise TypeError('old sha1 for %s is not a bytestring: %r' % (refname, old_sha1)) new_sha1 = new_refs.get(refname, ZERO_SHA) if not isinstance(new_sha1, bytes): raise TypeError('old sha1 for %s is not a bytestring %r' % (refname, new_sha1)) if old_sha1 != new_sha1: if sent_capabilities: proto.write_pkt_line(old_sha1 + b' ' + new_sha1 + b' ' + refname) else: proto.write_pkt_line( old_sha1 + b' ' + new_sha1 + b' ' + refname + b'\0' + b' '.join(capabilities)) sent_capabilities = True if new_sha1 not in have and new_sha1 != ZERO_SHA: want.append(new_sha1) proto.write_pkt_line(None) return (have, want) def _negotiate_receive_pack_capabilities(self, server_capabilities): negotiated_capabilities = ( self._send_capabilities & server_capabilities) unknown_capabilities = ( # noqa: F841 extract_capability_names(server_capabilities) - KNOWN_RECEIVE_CAPABILITIES) # TODO(jelmer): warn about unknown capabilities return negotiated_capabilities def _handle_receive_pack_tail(self, proto, capabilities, progress=None): """Handle the tail of a 'git-receive-pack' request. :param proto: Protocol object to read from :param capabilities: List of negotiated capabilities :param progress: Optional progress reporting function """ if CAPABILITY_SIDE_BAND_64K in capabilities: if progress is None: def progress(x): pass channel_callbacks = {2: progress} if CAPABILITY_REPORT_STATUS in capabilities: channel_callbacks[1] = PktLineParser( self._report_status_parser.handle_packet).parse self._read_side_band64k_data(proto, channel_callbacks) else: if CAPABILITY_REPORT_STATUS in capabilities: for pkt in proto.read_pkt_seq(): self._report_status_parser.handle_packet(pkt) if self._report_status_parser is not None: self._report_status_parser.check() def _negotiate_upload_pack_capabilities(self, server_capabilities): unknown_capabilities = ( # noqa: F841 extract_capability_names(server_capabilities) - KNOWN_UPLOAD_CAPABILITIES) # TODO(jelmer): warn about unknown capabilities symrefs = {} agent = None for capability in server_capabilities: k, v = parse_capability(capability) if k == CAPABILITY_SYMREF: (src, dst) = v.split(b':', 1) symrefs[src] = dst if k == CAPABILITY_AGENT: agent = v negotiated_capabilities = ( self._fetch_capabilities & server_capabilities) return (negotiated_capabilities, symrefs, agent) def _handle_upload_pack_head(self, proto, capabilities, graph_walker, - wants, can_read): + wants, can_read, depth): """Handle the head of a 'git-upload-pack' request. :param proto: Protocol object to read from :param capabilities: List of negotiated capabilities :param graph_walker: GraphWalker instance to call .ack() on :param wants: List of commits to fetch :param can_read: function that returns a boolean that indicates whether there is extra graph data to read on proto + :param depth: Depth for request """ assert isinstance(wants, list) and isinstance(wants[0], bytes) proto.write_pkt_line(COMMAND_WANT + b' ' + wants[0] + b' ' + b' '.join(capabilities) + b'\n') for want in wants[1:]: proto.write_pkt_line(COMMAND_WANT + b' ' + want + b'\n') + if depth not in (0, None): + if not CAPABILITY_SHALLOW in capabilities: + raise GitProtocolError( + "server does not support shallow capability required for " + "depth") + proto.write_pkt_line(b'%s %d\n' % (COMMAND_DEEPEN, depth)) proto.write_pkt_line(None) have = next(graph_walker) while have: proto.write_pkt_line(COMMAND_HAVE + b' ' + have + b'\n') if can_read(): pkt = proto.read_pkt_line() parts = pkt.rstrip(b'\n').split(b' ') if parts[0] == b'ACK': graph_walker.ack(parts[1]) if parts[2] in (b'continue', b'common'): pass elif parts[2] == b'ready': break else: raise AssertionError( "%s not in ('continue', 'ready', 'common)" % parts[2]) have = next(graph_walker) proto.write_pkt_line(COMMAND_DONE + b'\n') def _handle_upload_pack_tail(self, proto, capabilities, graph_walker, pack_data, progress=None, rbufsize=_RBUFSIZE): """Handle the tail of a 'git-upload-pack' request. :param proto: Protocol object to read from :param capabilities: List of negotiated capabilities :param graph_walker: GraphWalker instance to call .ack() on :param pack_data: Function to call with pack data :param progress: Optional progress reporting function :param rbufsize: Read buffer size """ pkt = proto.read_pkt_line() while pkt: parts = pkt.rstrip(b'\n').split(b' ') if parts[0] == b'ACK': graph_walker.ack(parts[1]) if len(parts) < 3 or parts[2] not in ( b'ready', b'continue', b'common'): break pkt = proto.read_pkt_line() if CAPABILITY_SIDE_BAND_64K in capabilities: if progress is None: # Just ignore progress data def progress(x): pass self._read_side_band64k_data(proto, { SIDE_BAND_CHANNEL_DATA: pack_data, SIDE_BAND_CHANNEL_PROGRESS: progress} ) else: while True: data = proto.read(rbufsize) if data == b"": break pack_data(data) def check_wants(wants, refs): """Check that a set of wants is valid. :param wants: Set of object SHAs to fetch :param refs: Refs dictionary to check against """ missing = set(wants) - { v for (k, v) in refs.items() if not k.endswith(ANNOTATED_TAG_SUFFIX)} if missing: raise InvalidWants(missing) def remote_error_from_stderr(stderr): """Return an appropriate exception based on stderr output. """ if stderr is None: return HangupException() for l in stderr.readlines(): if l.startswith(b'ERROR: '): return GitProtocolError( l[len(b'ERROR: '):].decode('utf-8', 'replace')) return GitProtocolError(l.decode('utf-8', 'replace')) return HangupException() class TraditionalGitClient(GitClient): """Traditional Git client.""" DEFAULT_ENCODING = 'utf-8' def __init__(self, path_encoding=DEFAULT_ENCODING, **kwargs): self._remote_path_encoding = path_encoding super(TraditionalGitClient, self).__init__(**kwargs) def _connect(self, cmd, path): """Create a connection to the server. This method is abstract - concrete implementations should implement their own variant which connects to the server and returns an initialized Protocol object with the service ready for use and a can_read function which may be used to see if reads would block. :param cmd: The git service name to which we should connect. :param path: The path we should pass to the service. (as bytestirng) """ raise NotImplementedError() def send_pack(self, path, update_refs, generate_pack_data, progress=None): """Upload a pack to a remote repository. :param path: Repository path (as bytestring) :param update_refs: Function to determine changes to remote refs. Receive dict with existing remote refs, returns dict with changed refs (name -> sha, where sha=ZERO_SHA for deletions) :param generate_pack_data: Function that can return a tuple with number of objects and pack data to upload. :param progress: Optional callback called with progress updates :raises SendPackError: if server rejects the pack data :raises UpdateRefsError: if the server supports report-status and rejects ref updates :return: new_refs dictionary containing the changes that were made {refname: new_ref}, including deleted refs. """ proto, unused_can_read, stderr = self._connect(b'receive-pack', path) with proto: try: old_refs, server_capabilities = read_pkt_refs(proto) except HangupException: raise remote_error_from_stderr(stderr) negotiated_capabilities = \ self._negotiate_receive_pack_capabilities(server_capabilities) if CAPABILITY_REPORT_STATUS in negotiated_capabilities: self._report_status_parser = ReportStatusParser() report_status_parser = self._report_status_parser try: new_refs = orig_new_refs = update_refs(dict(old_refs)) except BaseException: proto.write_pkt_line(None) raise if CAPABILITY_DELETE_REFS not in server_capabilities: # Server does not support deletions. Fail later. new_refs = dict(orig_new_refs) for ref, sha in orig_new_refs.items(): if sha == ZERO_SHA: if CAPABILITY_REPORT_STATUS in negotiated_capabilities: report_status_parser._ref_statuses.append( b'ng ' + sha + b' remote does not support deleting refs') report_status_parser._ref_status_ok = False del new_refs[ref] if new_refs is None: proto.write_pkt_line(None) return old_refs if len(new_refs) == 0 and len(orig_new_refs): # NOOP - Original new refs filtered out by policy proto.write_pkt_line(None) if report_status_parser is not None: report_status_parser.check() return old_refs (have, want) = self._handle_receive_pack_head( proto, negotiated_capabilities, old_refs, new_refs) if (not want and set(new_refs.items()).issubset(set(old_refs.items()))): return new_refs pack_data_count, pack_data = generate_pack_data( have, want, ofs_delta=(CAPABILITY_OFS_DELTA in negotiated_capabilities)) dowrite = bool(pack_data_count) dowrite = dowrite or any(old_refs.get(ref) != sha for (ref, sha) in new_refs.items() if sha != ZERO_SHA) if dowrite: write_pack_data(proto.write_file(), pack_data_count, pack_data) self._handle_receive_pack_tail( proto, negotiated_capabilities, progress) return new_refs def fetch_pack(self, path, determine_wants, graph_walker, pack_data, - progress=None): + depth=None, progress=None): """Retrieve a pack from a git smart server. :param path: Remote path to fetch from :param determine_wants: Function determine what refs to fetch. Receives dictionary of name->sha, should return list of shas to fetch. :param graph_walker: Object with next() and ack(). :param pack_data: Callback called for each bit of data in the pack + :param depth: Depth for request :param progress: Callback for progress reports (strings) :return: FetchPackResult object """ proto, can_read, stderr = self._connect(b'upload-pack', path) with proto: try: refs, server_capabilities = read_pkt_refs(proto) except HangupException: raise remote_error_from_stderr(stderr) negotiated_capabilities, symrefs, agent = ( self._negotiate_upload_pack_capabilities( server_capabilities)) if refs is None: proto.write_pkt_line(None) return FetchPackResult(refs, symrefs, agent) try: wants = determine_wants(refs) except BaseException: proto.write_pkt_line(None) raise if wants is not None: wants = [cid for cid in wants if cid != ZERO_SHA] if not wants: proto.write_pkt_line(None) return FetchPackResult(refs, symrefs, agent) check_wants(wants, refs) self._handle_upload_pack_head( - proto, negotiated_capabilities, graph_walker, wants, can_read) + proto, negotiated_capabilities, graph_walker, wants, can_read, + depth=depth) self._handle_upload_pack_tail( proto, negotiated_capabilities, graph_walker, pack_data, progress) return FetchPackResult(refs, symrefs, agent) def get_refs(self, path): """Retrieve the current refs from a git smart server.""" # stock `git ls-remote` uses upload-pack proto, _, stderr = self._connect(b'upload-pack', path) with proto: try: refs, _ = read_pkt_refs(proto) except HangupException: raise remote_error_from_stderr(stderr) proto.write_pkt_line(None) return refs def archive(self, path, committish, write_data, progress=None, write_error=None, format=None, subdirs=None, prefix=None): proto, can_read, stderr = self._connect(b'upload-archive', path) with proto: if format is not None: proto.write_pkt_line(b"argument --format=" + format) proto.write_pkt_line(b"argument " + committish) if subdirs is not None: for subdir in subdirs: proto.write_pkt_line(b"argument " + subdir) if prefix is not None: proto.write_pkt_line(b"argument --prefix=" + prefix) proto.write_pkt_line(None) try: pkt = proto.read_pkt_line() except HangupException: raise remote_error_from_stderr(stderr) if pkt == b"NACK\n": return elif pkt == b"ACK\n": pass elif pkt.startswith(b"ERR "): raise GitProtocolError( pkt[4:].rstrip(b"\n").decode('utf-8', 'replace')) else: raise AssertionError("invalid response %r" % pkt) ret = proto.read_pkt_line() if ret is not None: raise AssertionError("expected pkt tail") self._read_side_band64k_data(proto, { SIDE_BAND_CHANNEL_DATA: write_data, SIDE_BAND_CHANNEL_PROGRESS: progress, SIDE_BAND_CHANNEL_FATAL: write_error}) class TCPGitClient(TraditionalGitClient): """A Git Client that works over TCP directly (i.e. git://).""" def __init__(self, host, port=None, **kwargs): if port is None: port = TCP_GIT_PORT self._host = host self._port = port super(TCPGitClient, self).__init__(**kwargs) @classmethod def from_parsedurl(cls, parsedurl, **kwargs): return cls(parsedurl.hostname, port=parsedurl.port, **kwargs) def get_url(self, path): netloc = self._host if self._port is not None and self._port != TCP_GIT_PORT: netloc += ":%d" % self._port return urlparse.urlunsplit(("git", netloc, path, '', '')) def _connect(self, cmd, path): if not isinstance(cmd, bytes): raise TypeError(cmd) if not isinstance(path, bytes): path = path.encode(self._remote_path_encoding) sockaddrs = socket.getaddrinfo( self._host, self._port, socket.AF_UNSPEC, socket.SOCK_STREAM) s = None err = socket.error("no address found for %s" % self._host) for (family, socktype, proto, canonname, sockaddr) in sockaddrs: s = socket.socket(family, socktype, proto) s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) try: s.connect(sockaddr) break except socket.error as e: err = e if s is not None: s.close() s = None if s is None: raise err # -1 means system default buffering rfile = s.makefile('rb', -1) # 0 means unbuffered wfile = s.makefile('wb', 0) def close(): rfile.close() wfile.close() s.close() proto = Protocol(rfile.read, wfile.write, close, report_activity=self._report_activity) if path.startswith(b"/~"): path = path[1:] # TODO(jelmer): Alternative to ascii? proto.send_cmd( b'git-' + cmd, path, b'host=' + self._host.encode('ascii')) return proto, lambda: _fileno_can_read(s), None class SubprocessWrapper(object): """A socket-like object that talks to a subprocess via pipes.""" def __init__(self, proc): self.proc = proc if sys.version_info[0] == 2: self.read = proc.stdout.read else: self.read = BufferedReader(proc.stdout).read self.write = proc.stdin.write @property def stderr(self): return self.proc.stderr def can_read(self): if sys.platform == 'win32': from msvcrt import get_osfhandle handle = get_osfhandle(self.proc.stdout.fileno()) return _win32_peek_avail(handle) != 0 else: return _fileno_can_read(self.proc.stdout.fileno()) def close(self): self.proc.stdin.close() self.proc.stdout.close() if self.proc.stderr: self.proc.stderr.close() self.proc.wait() def find_git_command(): """Find command to run for system Git (usually C Git). """ if sys.platform == 'win32': # support .exe, .bat and .cmd try: # to avoid overhead import win32api except ImportError: # run through cmd.exe with some overhead return ['cmd', '/c', 'git'] else: status, git = win32api.FindExecutable('git') return [git] else: return ['git'] class SubprocessGitClient(TraditionalGitClient): """Git client that talks to a server using a subprocess.""" @classmethod def from_parsedurl(cls, parsedurl, **kwargs): return cls(**kwargs) git_command = None def _connect(self, service, path): if not isinstance(service, bytes): raise TypeError(service) if isinstance(path, bytes): path = path.decode(self._remote_path_encoding) if self.git_command is None: git_command = find_git_command() argv = git_command + [service.decode('ascii'), path] p = subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pw = SubprocessWrapper(p) return (Protocol(pw.read, pw.write, pw.close, report_activity=self._report_activity), pw.can_read, p.stderr) class LocalGitClient(GitClient): """Git Client that just uses a local Repo.""" def __init__(self, thin_packs=True, report_activity=None, config=None): """Create a new LocalGitClient instance. :param thin_packs: Whether or not thin packs should be retrieved :param report_activity: Optional callback for reporting transport activity. """ self._report_activity = report_activity # Ignore the thin_packs argument def get_url(self, path): return urlparse.urlunsplit(('file', '', path, '', '')) @classmethod def from_parsedurl(cls, parsedurl, **kwargs): return cls(**kwargs) @classmethod def _open_repo(cls, path): from dulwich.repo import Repo if not isinstance(path, str): path = path.decode(sys.getfilesystemencoding()) return closing(Repo(path)) def send_pack(self, path, update_refs, generate_pack_data, progress=None): """Upload a pack to a remote repository. :param path: Repository path (as bytestring) :param update_refs: Function to determine changes to remote refs. Receive dict with existing remote refs, returns dict with changed refs (name -> sha, where sha=ZERO_SHA for deletions) :param generate_pack_data: Function that can return a tuple with number of items and pack data to upload. :param progress: Optional progress function :raises SendPackError: if server rejects the pack data :raises UpdateRefsError: if the server supports report-status and rejects ref updates :return: new_refs dictionary containing the changes that were made {refname: new_ref}, including deleted refs. """ if not progress: def progress(x): pass with self._open_repo(path) as target: old_refs = target.get_refs() new_refs = update_refs(dict(old_refs)) have = [sha1 for sha1 in old_refs.values() if sha1 != ZERO_SHA] want = [] for refname, new_sha1 in new_refs.items(): if (new_sha1 not in have and new_sha1 not in want and new_sha1 != ZERO_SHA): want.append(new_sha1) if (not want and set(new_refs.items()).issubset(set(old_refs.items()))): return new_refs target.object_store.add_pack_data( *generate_pack_data(have, want, ofs_delta=True)) for refname, new_sha1 in new_refs.items(): old_sha1 = old_refs.get(refname, ZERO_SHA) if new_sha1 != ZERO_SHA: if not target.refs.set_if_equals( refname, old_sha1, new_sha1): progress('unable to set %s to %s' % (refname, new_sha1)) else: if not target.refs.remove_if_equals(refname, old_sha1): progress('unable to remove %s' % refname) return new_refs def fetch(self, path, target, determine_wants=None, progress=None): """Fetch into a target repository. :param path: Path to fetch from (as bytestring) :param target: Target repository to fetch into :param determine_wants: Optional function determine what refs to fetch. Receives dictionary of name->sha, should return list of shas to fetch. Defaults to all shas. :param progress: Optional progress function :return: FetchPackResult object """ with self._open_repo(path) as r: refs = r.fetch(target, determine_wants=determine_wants, progress=progress) return FetchPackResult(refs, r.refs.get_symrefs(), agent_string()) def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress=None): """Retrieve a pack from a git smart server. :param path: Remote path to fetch from :param determine_wants: Function determine what refs to fetch. Receives dictionary of name->sha, should return list of shas to fetch. :param graph_walker: Object with next() and ack(). :param pack_data: Callback called for each bit of data in the pack :param progress: Callback for progress reports (strings) :return: FetchPackResult object """ with self._open_repo(path) as r: objects_iter = r.fetch_objects( determine_wants, graph_walker, progress) symrefs = r.refs.get_symrefs() agent = agent_string() # Did the process short-circuit (e.g. in a stateless RPC call)? # Note that the client still expects a 0-object pack in most cases. if objects_iter is None: return FetchPackResult(None, symrefs, agent) protocol = ProtocolFile(None, pack_data) write_pack_objects(protocol, objects_iter) return FetchPackResult(r.get_refs(), symrefs, agent) def get_refs(self, path): """Retrieve the current refs from a git smart server.""" with self._open_repo(path) as target: return target.get_refs() # What Git client to use for local access default_local_git_client_cls = LocalGitClient class SSHVendor(object): """A client side SSH implementation.""" def connect_ssh(self, host, command, username=None, port=None, password=None, key_filename=None): # This function was deprecated in 0.9.1 import warnings warnings.warn( "SSHVendor.connect_ssh has been renamed to SSHVendor.run_command", DeprecationWarning) return self.run_command(host, command, username=username, port=port, password=password, key_filename=key_filename) def run_command(self, host, command, username=None, port=None, password=None, key_filename=None): """Connect to an SSH server. Run a command remotely and return a file-like object for interaction with the remote command. :param host: Host name :param command: Command to run (as argv array) :param username: Optional ame of user to log in as :param port: Optional SSH port to use :param password: Optional ssh password for login or private key :param key_filename: Optional path to private keyfile """ raise NotImplementedError(self.run_command) class StrangeHostname(Exception): """Refusing to connect to strange SSH hostname.""" def __init__(self, hostname): super(StrangeHostname, self).__init__(hostname) class SubprocessSSHVendor(SSHVendor): """SSH vendor that shells out to the local 'ssh' command.""" def run_command(self, host, command, username=None, port=None, password=None, key_filename=None): if password is not None: raise NotImplementedError( "Setting password not supported by SubprocessSSHVendor.") args = ['ssh', '-x'] if port: args.extend(['-p', str(port)]) if key_filename: args.extend(['-i', str(key_filename)]) if username: host = '%s@%s' % (username, host) if host.startswith('-'): raise StrangeHostname(hostname=host) args.append(host) proc = subprocess.Popen(args + [command], bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return SubprocessWrapper(proc) class PLinkSSHVendor(SSHVendor): """SSH vendor that shells out to the local 'plink' command.""" def run_command(self, host, command, username=None, port=None, password=None, key_filename=None): if sys.platform == 'win32': args = ['plink.exe', '-ssh'] else: args = ['plink', '-ssh'] if password is not None: import warnings warnings.warn( "Invoking PLink with a password exposes the password in the " "process list.") args.extend(['-pw', str(password)]) if port: args.extend(['-P', str(port)]) if key_filename: args.extend(['-i', str(key_filename)]) if username: host = '%s@%s' % (username, host) if host.startswith('-'): raise StrangeHostname(hostname=host) args.append(host) proc = subprocess.Popen(args + [command], bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return SubprocessWrapper(proc) def ParamikoSSHVendor(**kwargs): import warnings warnings.warn( "ParamikoSSHVendor has been moved to dulwich.contrib.paramiko_vendor.", DeprecationWarning) from dulwich.contrib.paramiko_vendor import ParamikoSSHVendor return ParamikoSSHVendor(**kwargs) # Can be overridden by users get_ssh_vendor = SubprocessSSHVendor class SSHGitClient(TraditionalGitClient): def __init__(self, host, port=None, username=None, vendor=None, config=None, password=None, key_filename=None, **kwargs): self.host = host self.port = port self.username = username self.password = password self.key_filename = key_filename super(SSHGitClient, self).__init__(**kwargs) self.alternative_paths = {} if vendor is not None: self.ssh_vendor = vendor else: self.ssh_vendor = get_ssh_vendor() def get_url(self, path): netloc = self.host if self.port is not None: netloc += ":%d" % self.port if self.username is not None: netloc = urlquote(self.username, '@/:') + "@" + netloc return urlparse.urlunsplit(('ssh', netloc, path, '', '')) @classmethod def from_parsedurl(cls, parsedurl, **kwargs): return cls(host=parsedurl.hostname, port=parsedurl.port, username=parsedurl.username, **kwargs) def _get_cmd_path(self, cmd): cmd = self.alternative_paths.get(cmd, b'git-' + cmd) assert isinstance(cmd, bytes) return cmd def _connect(self, cmd, path): if not isinstance(cmd, bytes): raise TypeError(cmd) if isinstance(path, bytes): path = path.decode(self._remote_path_encoding) if path.startswith("/~"): path = path[1:] argv = (self._get_cmd_path(cmd).decode(self._remote_path_encoding) + " '" + path + "'") kwargs = {} if self.password is not None: kwargs['password'] = self.password if self.key_filename is not None: kwargs['key_filename'] = self.key_filename con = self.ssh_vendor.run_command( self.host, argv, port=self.port, username=self.username, **kwargs) return (Protocol(con.read, con.write, con.close, report_activity=self._report_activity), con.can_read, getattr(con, 'stderr', None)) def default_user_agent_string(): # Start user agent with "git/", because GitHub requires this. :-( See # https://github.com/jelmer/dulwich/issues/562 for details. return "git/dulwich/%s" % ".".join([str(x) for x in dulwich.__version__]) def default_urllib3_manager(config, **override_kwargs): """Return `urllib3` connection pool manager. Honour detected proxy configurations. :param config: `dulwich.config.ConfigDict` instance with Git configuration. :param kwargs: Additional arguments for urllib3.ProxyManager :return: `urllib3.ProxyManager` instance for proxy configurations, `urllib3.PoolManager` otherwise. """ proxy_server = user_agent = None ca_certs = ssl_verify = None if config is not None: try: proxy_server = config.get(b"http", b"proxy") except KeyError: pass try: user_agent = config.get(b"http", b"useragent") except KeyError: pass # TODO(jelmer): Support per-host settings try: ssl_verify = config.get_boolean(b"http", b"sslVerify") except KeyError: ssl_verify = True try: ca_certs = config.get_boolean(b"http", b"sslCAInfo") except KeyError: ca_certs = None if user_agent is None: user_agent = default_user_agent_string() headers = {"User-agent": user_agent} kwargs = {} if ssl_verify is True: kwargs['cert_reqs'] = "CERT_REQUIRED" elif ssl_verify is False: kwargs['cert_reqs'] = 'CERT_NONE' else: # Default to SSL verification kwargs['cert_reqs'] = "CERT_REQUIRED" if ca_certs is not None: kwargs['ca_certs'] = ca_certs kwargs.update(override_kwargs) # Try really hard to find a SSL certificate path if 'ca_certs' not in kwargs and kwargs.get('cert_reqs') != 'CERT_NONE': try: import certifi except ImportError: pass else: kwargs['ca_certs'] = certifi.where() import urllib3 if proxy_server is not None: # `urllib3` requires a `str` object in both Python 2 and 3, while # `ConfigDict` coerces entries to `bytes` on Python 3. Compensate. if not isinstance(proxy_server, str): proxy_server = proxy_server.decode() manager = urllib3.ProxyManager(proxy_server, headers=headers, **kwargs) else: manager = urllib3.PoolManager(headers=headers, **kwargs) return manager class HttpGitClient(GitClient): def __init__(self, base_url, dumb=None, pool_manager=None, config=None, username=None, password=None, **kwargs): self._base_url = base_url.rstrip("/") + "/" self._username = username self._password = password self.dumb = dumb if pool_manager is None: self.pool_manager = default_urllib3_manager(config) else: self.pool_manager = pool_manager if username is not None: # No escaping needed: ":" is not allowed in username: # https://tools.ietf.org/html/rfc2617#section-2 credentials = "%s:%s" % (username, password) import urllib3.util basic_auth = urllib3.util.make_headers(basic_auth=credentials) self.pool_manager.headers.update(basic_auth) GitClient.__init__(self, **kwargs) def get_url(self, path): return self._get_url(path).rstrip("/") @classmethod def from_parsedurl(cls, parsedurl, **kwargs): password = parsedurl.password if password is not None: kwargs['password'] = urlunquote(password) username = parsedurl.username if username is not None: kwargs['username'] = urlunquote(username) # TODO(jelmer): This also strips the username parsedurl = parsedurl._replace(netloc=parsedurl.hostname) return cls(urlparse.urlunparse(parsedurl), **kwargs) def __repr__(self): return "%s(%r, dumb=%r)" % ( type(self).__name__, self._base_url, self.dumb) def _get_url(self, path): if not isinstance(path, str): # TODO(jelmer): this is unrelated to the local filesystem; # This is not necessarily the right encoding to decode the path # with. path = path.decode(sys.getfilesystemencoding()) return urlparse.urljoin(self._base_url, path).rstrip("/") + "/" def _http_request(self, url, headers=None, data=None, allow_compression=False): """Perform HTTP request. :param url: Request URL. :param headers: Optional custom headers to override defaults. :param data: Request data. :param allow_compression: Allow GZipped communication. :return: Tuple (`response`, `read`), where response is an `urllib3` response object with additional `content_type` and `redirect_location` properties, and `read` is a consumable read method for the response data. """ req_headers = self.pool_manager.headers.copy() if headers is not None: req_headers.update(headers) req_headers["Pragma"] = "no-cache" if allow_compression: req_headers["Accept-Encoding"] = "gzip" else: req_headers["Accept-Encoding"] = "identity" if data is None: resp = self.pool_manager.request("GET", url, headers=req_headers) else: resp = self.pool_manager.request("POST", url, headers=req_headers, body=data) if resp.status == 404: raise NotGitRepository() elif resp.status != 200: raise GitProtocolError("unexpected http resp %d for %s" % (resp.status, url)) # TODO: Optimization available by adding `preload_content=False` to the # request and just passing the `read` method on instead of going via # `BytesIO`, if we can guarantee that the entire response is consumed # before issuing the next to still allow for connection reuse from the # pool. read = BytesIO(resp.data).read resp.content_type = resp.getheader("Content-Type") resp.redirect_location = resp.get_redirect_location() return resp, read def _discover_references(self, service, base_url): assert base_url[-1] == "/" tail = "info/refs" headers = {"Accept": "*/*"} if self.dumb is not True: tail += "?service=%s" % service.decode('ascii') url = urlparse.urljoin(base_url, tail) resp, read = self._http_request(url, headers, allow_compression=True) if resp.redirect_location: # Something changed (redirect!), so let's update the base URL if not resp.redirect_location.endswith(tail): raise GitProtocolError( "Redirected from URL %s to URL %s without %s" % ( url, resp.redirect_location, tail)) base_url = resp.redirect_location[:-len(tail)] try: self.dumb = not resp.content_type.startswith("application/x-git-") if not self.dumb: proto = Protocol(read, None) # The first line should mention the service try: [pkt] = list(proto.read_pkt_seq()) except ValueError: raise GitProtocolError( "unexpected number of packets received") if pkt.rstrip(b'\n') != (b'# service=' + service): raise GitProtocolError( "unexpected first line %r from smart server" % pkt) return read_pkt_refs(proto) + (base_url, ) else: return read_info_refs(resp), set(), base_url finally: resp.close() def _smart_request(self, service, url, data): assert url[-1] == "/" url = urlparse.urljoin(url, service) result_content_type = "application/x-%s-result" % service headers = { "Content-Type": "application/x-%s-request" % service, "Accept": result_content_type, "Content-Length": str(len(data)), } resp, read = self._http_request(url, headers, data) if resp.content_type != result_content_type: raise GitProtocolError("Invalid content-type from server: %s" % resp.content_type) return resp, read def send_pack(self, path, update_refs, generate_pack_data, progress=None): """Upload a pack to a remote repository. :param path: Repository path (as bytestring) :param update_refs: Function to determine changes to remote refs. Receive dict with existing remote refs, returns dict with changed refs (name -> sha, where sha=ZERO_SHA for deletions) :param generate_pack_data: Function that can return a tuple with number of elements and pack data to upload. :param progress: Optional progress function :raises SendPackError: if server rejects the pack data :raises UpdateRefsError: if the server supports report-status and rejects ref updates :return: new_refs dictionary containing the changes that were made {refname: new_ref}, including deleted refs. """ url = self._get_url(path) old_refs, server_capabilities, url = self._discover_references( b"git-receive-pack", url) negotiated_capabilities = self._negotiate_receive_pack_capabilities( server_capabilities) negotiated_capabilities.add(capability_agent()) if CAPABILITY_REPORT_STATUS in negotiated_capabilities: self._report_status_parser = ReportStatusParser() new_refs = update_refs(dict(old_refs)) if new_refs is None: # Determine wants function is aborting the push. return old_refs if self.dumb: raise NotImplementedError(self.fetch_pack) req_data = BytesIO() req_proto = Protocol(None, req_data.write) (have, want) = self._handle_receive_pack_head( req_proto, negotiated_capabilities, old_refs, new_refs) if not want and set(new_refs.items()).issubset(set(old_refs.items())): return new_refs pack_data_count, pack_data = generate_pack_data( have, want, ofs_delta=(CAPABILITY_OFS_DELTA in negotiated_capabilities)) if pack_data_count: write_pack_data(req_proto.write_file(), pack_data_count, pack_data) resp, read = self._smart_request("git-receive-pack", url, data=req_data.getvalue()) try: resp_proto = Protocol(read, None) self._handle_receive_pack_tail( resp_proto, negotiated_capabilities, progress) return new_refs finally: resp.close() def fetch_pack(self, path, determine_wants, graph_walker, pack_data, - progress=None): + depth=None, progress=None): """Retrieve a pack from a git smart server. :param determine_wants: Callback that returns list of commits to fetch :param graph_walker: Object with next() and ack(). :param pack_data: Callback called for each bit of data in the pack :param progress: Callback for progress reports (strings) + :param depth: Depth for request :return: FetchPackResult object """ url = self._get_url(path) refs, server_capabilities, url = self._discover_references( b"git-upload-pack", url) negotiated_capabilities, symrefs, agent = ( self._negotiate_upload_pack_capabilities( server_capabilities)) wants = determine_wants(refs) if wants is not None: wants = [cid for cid in wants if cid != ZERO_SHA] if not wants: return FetchPackResult(refs, symrefs, agent) if self.dumb: raise NotImplementedError(self.send_pack) check_wants(wants, refs) req_data = BytesIO() req_proto = Protocol(None, req_data.write) self._handle_upload_pack_head( req_proto, negotiated_capabilities, graph_walker, wants, - lambda: False) + lambda: False, depth=depth) resp, read = self._smart_request( "git-upload-pack", url, data=req_data.getvalue()) try: resp_proto = Protocol(read, None) self._handle_upload_pack_tail( resp_proto, negotiated_capabilities, graph_walker, pack_data, progress) return FetchPackResult(refs, symrefs, agent) finally: resp.close() def get_refs(self, path): """Retrieve the current refs from a git smart server.""" url = self._get_url(path) refs, _, _ = self._discover_references( b"git-upload-pack", url) return refs def get_transport_and_path_from_url(url, config=None, **kwargs): """Obtain a git client from a URL. :param url: URL to open (a unicode string) :param config: Optional config object :param thin_packs: Whether or not thin packs should be retrieved :param report_activity: Optional callback for reporting transport activity. :return: Tuple with client instance and relative path. """ parsed = urlparse.urlparse(url) if parsed.scheme == 'git': return (TCPGitClient.from_parsedurl(parsed, **kwargs), parsed.path) elif parsed.scheme in ('git+ssh', 'ssh'): return SSHGitClient.from_parsedurl(parsed, **kwargs), parsed.path elif parsed.scheme in ('http', 'https'): return HttpGitClient.from_parsedurl( parsed, config=config, **kwargs), parsed.path elif parsed.scheme == 'file': return default_local_git_client_cls.from_parsedurl( parsed, **kwargs), parsed.path raise ValueError("unknown scheme '%s'" % parsed.scheme) def parse_rsync_url(location): """Parse a rsync-style URL.""" if ':' in location and '@' not in location: # SSH with no user@, zero or one leading slash. (host, path) = location.split(':', 1) user = None elif ':' in location: # SSH with user@host:foo. user_host, path = location.split(':', 1) if '@' in user_host: user, host = user_host.rsplit('@', 1) else: user = None host = user_host else: raise ValueError('not a valid rsync-style URL') return (user, host, path) def get_transport_and_path(location, **kwargs): """Obtain a git client from a URL. :param location: URL or path (a string) :param config: Optional config object :param thin_packs: Whether or not thin packs should be retrieved :param report_activity: Optional callback for reporting transport activity. :return: Tuple with client instance and relative path. """ # First, try to parse it as a URL try: return get_transport_and_path_from_url(location, **kwargs) except ValueError: pass if (sys.platform == 'win32' and location[0].isalpha() and location[1:3] == ':\\'): # Windows local path return default_local_git_client_cls(**kwargs), location try: (username, hostname, path) = parse_rsync_url(location) except ValueError: # Otherwise, assume it's a local path. return default_local_git_client_cls(**kwargs), location else: return SSHGitClient(hostname, username=username, **kwargs), path diff --git a/dulwich/porcelain.py b/dulwich/porcelain.py index 0e1234b3..17aa3723 100644 --- a/dulwich/porcelain.py +++ b/dulwich/porcelain.py @@ -1,1409 +1,1412 @@ # porcelain.py -- Porcelain-like layer on top of Dulwich # Copyright (C) 2013 Jelmer Vernooij # # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU # General Public License as public by the Free Software Foundation; version 2.0 # or (at your option) any later version. You can redistribute it and/or # modify it under the terms of either of these two licenses. # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # You should have received a copy of the licenses; if not, see # for a copy of the GNU General Public License # and for a copy of the Apache # License, Version 2.0. # """Simple wrapper that provides porcelain-like functions on top of Dulwich. Currently implemented: * archive * add * branch{_create,_delete,_list} * check-ignore * checkout * clone * commit * commit-tree * daemon * describe * diff-tree * fetch * init * ls-files * ls-remote * ls-tree * pull * push * rm * remote{_add} * receive-pack * reset * rev-list * tag{_create,_delete,_list} * upload-pack * update-server-info * status * symbolic-ref These functions are meant to behave similarly to the git subcommands. Differences in behaviour are considered bugs. Functions should generally accept both unicode strings and bytestrings """ from collections import namedtuple from contextlib import ( closing, contextmanager, ) from io import BytesIO, RawIOBase import datetime import os import posixpath import stat import sys import time from dulwich.archive import ( tar_stream, ) from dulwich.client import ( get_transport_and_path, ) from dulwich.config import ( StackedConfig, ) from dulwich.diff_tree import ( CHANGE_ADD, CHANGE_DELETE, CHANGE_MODIFY, CHANGE_RENAME, CHANGE_COPY, RENAME_CHANGE_TYPES, ) from dulwich.errors import ( SendPackError, UpdateRefsError, ) from dulwich.ignore import IgnoreFilterManager from dulwich.index import ( blob_from_path_and_stat, get_unstaged_changes, ) from dulwich.object_store import ( tree_lookup_path, ) from dulwich.objects import ( Commit, Tag, format_timezone, parse_timezone, pretty_format_tree_entry, ) from dulwich.objectspec import ( parse_commit, parse_object, parse_ref, parse_reftuples, parse_tree, ) from dulwich.pack import ( write_pack_index, write_pack_objects, ) from dulwich.patch import write_tree_diff from dulwich.protocol import ( Protocol, ZERO_SHA, ) from dulwich.refs import ( ANNOTATED_TAG_SUFFIX, strip_peeled_refs, ) from dulwich.repo import (BaseRepo, Repo) from dulwich.server import ( FileSystemBackend, TCPGitServer, ReceivePackHandler, UploadPackHandler, update_server_info as server_update_server_info, ) # Module level tuple definition for status output GitStatus = namedtuple('GitStatus', 'staged unstaged untracked') class NoneStream(RawIOBase): """Fallback if stdout or stderr are unavailable, does nothing.""" def read(self, size=-1): return None def readall(self): return None def readinto(self, b): return None def write(self, b): return None default_bytes_out_stream = getattr( sys.stdout, 'buffer', sys.stdout ) or NoneStream() default_bytes_err_stream = getattr( sys.stderr, 'buffer', sys.stderr ) or NoneStream() DEFAULT_ENCODING = 'utf-8' class RemoteExists(Exception): """Raised when the remote already exists.""" def open_repo(path_or_repo): """Open an argument that can be a repository or a path for a repository.""" if isinstance(path_or_repo, BaseRepo): return path_or_repo return Repo(path_or_repo) @contextmanager def _noop_context_manager(obj): """Context manager that has the same api as closing but does nothing.""" yield obj def open_repo_closing(path_or_repo): """Open an argument that can be a repository or a path for a repository. returns a context manager that will close the repo on exit if the argument is a path, else does nothing if the argument is a repo. """ if isinstance(path_or_repo, BaseRepo): return _noop_context_manager(path_or_repo) return closing(Repo(path_or_repo)) def path_to_tree_path(repopath, path): """Convert a path to a path usable in an index, e.g. bytes and relative to the repository root. :param repopath: Repository path, absolute or relative to the cwd :param path: A path, absolute or relative to the cwd :return: A path formatted for use in e.g. an index """ if not isinstance(path, bytes): path = path.encode(sys.getfilesystemencoding()) if not isinstance(repopath, bytes): repopath = repopath.encode(sys.getfilesystemencoding()) treepath = os.path.relpath(path, repopath) if treepath.startswith(b'..'): raise ValueError('Path not in repo') if os.path.sep != '/': treepath = treepath.replace(os.path.sep.encode('ascii'), b'/') return treepath def archive(repo, committish=None, outstream=default_bytes_out_stream, errstream=default_bytes_err_stream): """Create an archive. :param repo: Path of repository for which to generate an archive. :param committish: Commit SHA1 or ref to use :param outstream: Output stream (defaults to stdout) :param errstream: Error stream (defaults to stderr) """ if committish is None: committish = "HEAD" with open_repo_closing(repo) as repo_obj: c = repo_obj[committish] for chunk in tar_stream( repo_obj.object_store, repo_obj.object_store[c.tree], c.commit_time): outstream.write(chunk) def update_server_info(repo="."): """Update server info files for a repository. :param repo: path to the repository """ with open_repo_closing(repo) as r: server_update_server_info(r) def symbolic_ref(repo, ref_name, force=False): """Set git symbolic ref into HEAD. :param repo: path to the repository :param ref_name: short name of the new ref :param force: force settings without checking if it exists in refs/heads """ with open_repo_closing(repo) as repo_obj: ref_path = _make_branch_ref(ref_name) if not force and ref_path not in repo_obj.refs.keys(): raise ValueError('fatal: ref `%s` is not a ref' % ref_name) repo_obj.refs.set_symbolic_ref(b'HEAD', ref_path) def commit(repo=".", message=None, author=None, committer=None, encoding=None): """Create a new commit. :param repo: Path to repository :param message: Optional commit message :param author: Optional author name and email :param committer: Optional committer name and email :return: SHA1 of the new commit """ # FIXME: Support --all argument # FIXME: Support --signoff argument if getattr(message, 'encode', None): message = message.encode(encoding or DEFAULT_ENCODING) if getattr(author, 'encode', None): author = author.encode(encoding or DEFAULT_ENCODING) if getattr(committer, 'encode', None): committer = committer.encode(encoding or DEFAULT_ENCODING) with open_repo_closing(repo) as r: return r.do_commit( message=message, author=author, committer=committer, encoding=encoding) def commit_tree(repo, tree, message=None, author=None, committer=None): """Create a new commit object. :param repo: Path to repository :param tree: An existing tree object :param author: Optional author name and email :param committer: Optional committer name and email """ with open_repo_closing(repo) as r: return r.do_commit( message=message, tree=tree, committer=committer, author=author) def init(path=".", bare=False): """Create a new git repository. :param path: Path to repository. :param bare: Whether to create a bare repository. :return: A Repo instance """ if not os.path.exists(path): os.mkdir(path) if bare: return Repo.init_bare(path) else: return Repo.init(path) def clone(source, target=None, bare=False, checkout=None, errstream=default_bytes_err_stream, outstream=None, - origin=b"origin", **kwargs): + origin=b"origin", depth=None, **kwargs): """Clone a local or remote git repository. :param source: Path or URL for source repository :param target: Path to target repository (optional) :param bare: Whether or not to create a bare repository :param checkout: Whether or not to check-out HEAD after cloning :param errstream: Optional stream to write progress to :param outstream: Optional stream to write progress to (deprecated) :param origin: Name of remote from the repository used to clone + :param depth: Depth to fetch at :return: The new repository """ # TODO(jelmer): This code overlaps quite a bit with Repo.clone if outstream is not None: import warnings warnings.warn( "outstream= has been deprecated in favour of errstream=.", DeprecationWarning, stacklevel=3) errstream = outstream if checkout is None: checkout = (not bare) if checkout and bare: raise ValueError("checkout and bare are incompatible") if target is None: target = source.split("/")[-1] if not os.path.exists(target): os.mkdir(target) if bare: r = Repo.init_bare(target) else: r = Repo.init(target) reflog_message = b'clone: from ' + source.encode('utf-8') try: fetch_result = fetch( r, source, origin, errstream=errstream, message=reflog_message, - **kwargs) + depth=depth, **kwargs) target_config = r.get_config() if not isinstance(source, bytes): source = source.encode(DEFAULT_ENCODING) target_config.set((b'remote', origin), b'url', source) target_config.set( (b'remote', origin), b'fetch', b'+refs/heads/*:refs/remotes/' + origin + b'/*') target_config.write_to_path() # TODO(jelmer): Support symref capability, # https://github.com/jelmer/dulwich/issues/485 try: head = r[fetch_result[b'HEAD']] except KeyError: head = None else: r[b'HEAD'] = head.id if checkout and not bare and head is not None: errstream.write(b'Checking out ' + head.id + b'\n') r.reset_index(head.tree) except BaseException: r.close() raise return r def add(repo=".", paths=None): """Add files to the staging area. :param repo: Repository for the files :param paths: Paths to add. No value passed stages all modified files. :return: Tuple with set of added files and ignored files """ ignored = set() with open_repo_closing(repo) as r: ignore_manager = IgnoreFilterManager.from_repo(r) if not paths: paths = list( get_untracked_paths(os.getcwd(), r.path, r.open_index())) relpaths = [] if not isinstance(paths, list): paths = [paths] for p in paths: relpath = os.path.relpath(p, r.path) if relpath.startswith('..' + os.path.sep): raise ValueError('path %r is not in repo' % relpath) # FIXME: Support patterns, directories. if ignore_manager.is_ignored(relpath): ignored.add(relpath) continue relpaths.append(relpath) r.stage(relpaths) return (relpaths, ignored) def remove(repo=".", paths=None, cached=False): """Remove files from the staging area. :param repo: Repository for the files :param paths: Paths to remove """ with open_repo_closing(repo) as r: index = r.open_index() for p in paths: full_path = os.path.abspath(p).encode(sys.getfilesystemencoding()) tree_path = path_to_tree_path(r.path, p) try: index_sha = index[tree_path].sha except KeyError: raise Exception('%s did not match any files' % p) if not cached: try: st = os.lstat(full_path) except OSError: pass else: try: blob = blob_from_path_and_stat(full_path, st) except IOError: pass else: try: committed_sha = tree_lookup_path( r.__getitem__, r[r.head()].tree, tree_path)[1] except KeyError: committed_sha = None if blob.id != index_sha and index_sha != committed_sha: raise Exception( 'file has staged content differing ' 'from both the file and head: %s' % p) if index_sha != committed_sha: raise Exception( 'file has staged changes: %s' % p) os.remove(full_path) del index[tree_path] index.write() rm = remove def commit_decode(commit, contents, default_encoding=DEFAULT_ENCODING): if commit.encoding is not None: return contents.decode(commit.encoding, "replace") return contents.decode(default_encoding, "replace") def print_commit(commit, decode, outstream=sys.stdout): """Write a human-readable commit log entry. :param commit: A `Commit` object :param outstream: A stream file to write to """ outstream.write("-" * 50 + "\n") outstream.write("commit: " + commit.id.decode('ascii') + "\n") if len(commit.parents) > 1: outstream.write( "merge: " + "...".join([c.decode('ascii') for c in commit.parents[1:]]) + "\n") outstream.write("Author: " + decode(commit.author) + "\n") if commit.author != commit.committer: outstream.write("Committer: " + decode(commit.committer) + "\n") time_tuple = time.gmtime(commit.author_time + commit.author_timezone) time_str = time.strftime("%a %b %d %Y %H:%M:%S", time_tuple) timezone_str = format_timezone(commit.author_timezone).decode('ascii') outstream.write("Date: " + time_str + " " + timezone_str + "\n") outstream.write("\n") outstream.write(decode(commit.message) + "\n") outstream.write("\n") def print_tag(tag, decode, outstream=sys.stdout): """Write a human-readable tag. :param tag: A `Tag` object :param decode: Function for decoding bytes to unicode string :param outstream: A stream to write to """ outstream.write("Tagger: " + decode(tag.tagger) + "\n") outstream.write("Date: " + decode(tag.tag_time) + "\n") outstream.write("\n") outstream.write(decode(tag.message) + "\n") outstream.write("\n") def show_blob(repo, blob, decode, outstream=sys.stdout): """Write a blob to a stream. :param repo: A `Repo` object :param blob: A `Blob` object :param decode: Function for decoding bytes to unicode string :param outstream: A stream file to write to """ outstream.write(decode(blob.data)) def show_commit(repo, commit, decode, outstream=sys.stdout): """Show a commit to a stream. :param repo: A `Repo` object :param commit: A `Commit` object :param decode: Function for decoding bytes to unicode string :param outstream: Stream to write to """ print_commit(commit, decode=decode, outstream=outstream) if commit.parents: parent_commit = repo[commit.parents[0]] base_tree = parent_commit.tree else: base_tree = None diffstream = BytesIO() write_tree_diff( diffstream, repo.object_store, base_tree, commit.tree) diffstream.seek(0) outstream.write( diffstream.getvalue().decode( commit.encoding or DEFAULT_ENCODING, 'replace')) def show_tree(repo, tree, decode, outstream=sys.stdout): """Print a tree to a stream. :param repo: A `Repo` object :param tree: A `Tree` object :param decode: Function for decoding bytes to unicode string :param outstream: Stream to write to """ for n in tree: outstream.write(decode(n) + "\n") def show_tag(repo, tag, decode, outstream=sys.stdout): """Print a tag to a stream. :param repo: A `Repo` object :param tag: A `Tag` object :param decode: Function for decoding bytes to unicode string :param outstream: Stream to write to """ print_tag(tag, decode, outstream) show_object(repo, repo[tag.object[1]], outstream) def show_object(repo, obj, decode, outstream): return { b"tree": show_tree, b"blob": show_blob, b"commit": show_commit, b"tag": show_tag, }[obj.type_name](repo, obj, decode, outstream) def print_name_status(changes): """Print a simple status summary, listing changed files. """ for change in changes: if not change: continue if isinstance(change, list): change = change[0] if change.type == CHANGE_ADD: path1 = change.new.path path2 = '' kind = 'A' elif change.type == CHANGE_DELETE: path1 = change.old.path path2 = '' kind = 'D' elif change.type == CHANGE_MODIFY: path1 = change.new.path path2 = '' kind = 'M' elif change.type in RENAME_CHANGE_TYPES: path1 = change.old.path path2 = change.new.path if change.type == CHANGE_RENAME: kind = 'R' elif change.type == CHANGE_COPY: kind = 'C' yield '%-8s%-20s%-20s' % (kind, path1, path2) def log(repo=".", paths=None, outstream=sys.stdout, max_entries=None, reverse=False, name_status=False): """Write commit logs. :param repo: Path to repository :param paths: Optional set of specific paths to print entries for :param outstream: Stream to write log output to :param reverse: Reverse order in which entries are printed :param name_status: Print name status :param max_entries: Optional maximum number of entries to display """ with open_repo_closing(repo) as r: walker = r.get_walker( max_entries=max_entries, paths=paths, reverse=reverse) for entry in walker: def decode(x): return commit_decode(entry.commit, x) print_commit(entry.commit, decode, outstream) if name_status: outstream.writelines( [l+'\n' for l in print_name_status(entry.changes())]) # TODO(jelmer): better default for encoding? def show(repo=".", objects=None, outstream=sys.stdout, default_encoding=DEFAULT_ENCODING): """Print the changes in a commit. :param repo: Path to repository :param objects: Objects to show (defaults to [HEAD]) :param outstream: Stream to write to :param default_encoding: Default encoding to use if none is set in the commit """ if objects is None: objects = ["HEAD"] if not isinstance(objects, list): objects = [objects] with open_repo_closing(repo) as r: for objectish in objects: o = parse_object(r, objectish) if isinstance(o, Commit): def decode(x): return commit_decode(o, x, default_encoding) else: def decode(x): return x.decode(default_encoding) show_object(r, o, decode, outstream) def diff_tree(repo, old_tree, new_tree, outstream=sys.stdout): """Compares the content and mode of blobs found via two tree objects. :param repo: Path to repository :param old_tree: Id of old tree :param new_tree: Id of new tree :param outstream: Stream to write to """ with open_repo_closing(repo) as r: write_tree_diff(outstream, r.object_store, old_tree, new_tree) def rev_list(repo, commits, outstream=sys.stdout): """Lists commit objects in reverse chronological order. :param repo: Path to repository :param commits: Commits over which to iterate :param outstream: Stream to write to """ with open_repo_closing(repo) as r: for entry in r.get_walker(include=[r[c].id for c in commits]): outstream.write(entry.commit.id + b"\n") def tag(*args, **kwargs): import warnings warnings.warn("tag has been deprecated in favour of tag_create.", DeprecationWarning) return tag_create(*args, **kwargs) def tag_create( repo, tag, author=None, message=None, annotated=False, objectish="HEAD", tag_time=None, tag_timezone=None): """Creates a tag in git via dulwich calls: :param repo: Path to repository :param tag: tag string :param author: tag author (optional, if annotated is set) :param message: tag message (optional) :param annotated: whether to create an annotated tag :param objectish: object the tag should point at, defaults to HEAD :param tag_time: Optional time for annotated tag :param tag_timezone: Optional timezone for annotated tag """ with open_repo_closing(repo) as r: object = parse_object(r, objectish) if annotated: # Create the tag object tag_obj = Tag() if author is None: # TODO(jelmer): Don't use repo private method. author = r._get_user_identity() tag_obj.tagger = author tag_obj.message = message tag_obj.name = tag tag_obj.object = (type(object), object.id) if tag_time is None: tag_time = int(time.time()) tag_obj.tag_time = tag_time if tag_timezone is None: # TODO(jelmer) Use current user timezone rather than UTC tag_timezone = 0 elif isinstance(tag_timezone, str): tag_timezone = parse_timezone(tag_timezone) tag_obj.tag_timezone = tag_timezone r.object_store.add_object(tag_obj) tag_id = tag_obj.id else: tag_id = object.id r.refs[_make_tag_ref(tag)] = tag_id def list_tags(*args, **kwargs): import warnings warnings.warn("list_tags has been deprecated in favour of tag_list.", DeprecationWarning) return tag_list(*args, **kwargs) def tag_list(repo, outstream=sys.stdout): """List all tags. :param repo: Path to repository :param outstream: Stream to write tags to """ with open_repo_closing(repo) as r: tags = sorted(r.refs.as_dict(b"refs/tags")) return tags def tag_delete(repo, name): """Remove a tag. :param repo: Path to repository :param name: Name of tag to remove """ with open_repo_closing(repo) as r: if isinstance(name, bytes): names = [name] elif isinstance(name, list): names = name else: raise TypeError("Unexpected tag name type %r" % name) for name in names: del r.refs[_make_tag_ref(name)] def reset(repo, mode, treeish="HEAD"): """Reset current HEAD to the specified state. :param repo: Path to repository :param mode: Mode ("hard", "soft", "mixed") :param treeish: Treeish to reset to """ if mode != "hard": raise ValueError("hard is the only mode currently supported") with open_repo_closing(repo) as r: tree = parse_tree(r, treeish) r.reset_index(tree.id) def push(repo, remote_location, refspecs, outstream=default_bytes_out_stream, errstream=default_bytes_err_stream, **kwargs): """Remote push with dulwich via dulwich.client :param repo: Path to repository :param remote_location: Location of the remote :param refspecs: Refs to push to remote :param outstream: A stream file to write output :param errstream: A stream file to write errors """ # Open the repo with open_repo_closing(repo) as r: # Get the client and path client, path = get_transport_and_path( remote_location, config=r.get_config_stack(), **kwargs) selected_refs = [] def update_refs(refs): selected_refs.extend(parse_reftuples(r.refs, refs, refspecs)) new_refs = {} # TODO: Handle selected_refs == {None: None} for (lh, rh, force) in selected_refs: if lh is None: new_refs[rh] = ZERO_SHA else: new_refs[rh] = r.refs[lh] return new_refs err_encoding = getattr(errstream, 'encoding', None) or DEFAULT_ENCODING remote_location_bytes = client.get_url(path).encode(err_encoding) try: client.send_pack( path, update_refs, generate_pack_data=r.object_store.generate_pack_data, progress=errstream.write) errstream.write( b"Push to " + remote_location_bytes + b" successful.\n") except (UpdateRefsError, SendPackError) as e: errstream.write(b"Push to " + remote_location_bytes + b" failed -> " + e.message.encode(err_encoding) + b"\n") def pull(repo, remote_location=None, refspecs=None, outstream=default_bytes_out_stream, errstream=default_bytes_err_stream, **kwargs): """Pull from remote via dulwich.client :param repo: Path to repository :param remote_location: Location of the remote :param refspec: refspecs to fetch :param outstream: A stream file to write to output :param errstream: A stream file to write to errors """ # Open the repo with open_repo_closing(repo) as r: if remote_location is None: # TODO(jelmer): Lookup 'remote' for current branch in config raise NotImplementedError( "looking up remote from branch config not supported yet") if refspecs is None: refspecs = [b"HEAD"] selected_refs = [] def determine_wants(remote_refs): selected_refs.extend( parse_reftuples(remote_refs, r.refs, refspecs)) return [remote_refs[lh] for (lh, rh, force) in selected_refs] client, path = get_transport_and_path( remote_location, config=r.get_config_stack(), **kwargs) fetch_result = client.fetch( path, r, progress=errstream.write, determine_wants=determine_wants) for (lh, rh, force) in selected_refs: r.refs[rh] = fetch_result.refs[lh] if selected_refs: r[b'HEAD'] = fetch_result.refs[selected_refs[0][1]] # Perform 'git checkout .' - syncs staged changes tree = r[b"HEAD"].tree r.reset_index(tree=tree) def status(repo=".", ignored=False): """Returns staged, unstaged, and untracked changes relative to the HEAD. :param repo: Path to repository or repository object :param ignored: Whether to include ignored files in `untracked` :return: GitStatus tuple, staged - list of staged paths (diff index/HEAD) unstaged - list of unstaged paths (diff index/working-tree) untracked - list of untracked, un-ignored & non-.git paths """ with open_repo_closing(repo) as r: # 1. Get status of staged tracked_changes = get_tree_changes(r) # 2. Get status of unstaged index = r.open_index() unstaged_changes = list(get_unstaged_changes(index, r.path)) ignore_manager = IgnoreFilterManager.from_repo(r) untracked_paths = get_untracked_paths(r.path, r.path, index) if ignored: untracked_changes = list(untracked_paths) else: untracked_changes = [ p for p in untracked_paths if not ignore_manager.is_ignored(p)] return GitStatus(tracked_changes, unstaged_changes, untracked_changes) def get_untracked_paths(frompath, basepath, index): """Get untracked paths. ;param frompath: Path to walk :param basepath: Path to compare to :param index: Index to check against """ # If nothing is specified, add all non-ignored files. for dirpath, dirnames, filenames in os.walk(frompath): # Skip .git and below. if '.git' in dirnames: dirnames.remove('.git') if dirpath != basepath: continue if '.git' in filenames: filenames.remove('.git') if dirpath != basepath: continue for filename in filenames: ap = os.path.join(dirpath, filename) ip = path_to_tree_path(basepath, ap) if ip not in index: yield os.path.relpath(ap, frompath) def get_tree_changes(repo): """Return add/delete/modify changes to tree by comparing index to HEAD. :param repo: repo path or object :return: dict with lists for each type of change """ with open_repo_closing(repo) as r: index = r.open_index() # Compares the Index to the HEAD & determines changes # Iterate through the changes and report add/delete/modify # TODO: call out to dulwich.diff_tree somehow. tracked_changes = { 'add': [], 'delete': [], 'modify': [], } try: tree_id = r[b'HEAD'].tree except KeyError: tree_id = None for change in index.changes_from_tree(r.object_store, tree_id): if not change[0][0]: tracked_changes['add'].append(change[0][1]) elif not change[0][1]: tracked_changes['delete'].append(change[0][0]) elif change[0][0] == change[0][1]: tracked_changes['modify'].append(change[0][0]) else: raise AssertionError('git mv ops not yet supported') return tracked_changes def daemon(path=".", address=None, port=None): """Run a daemon serving Git requests over TCP/IP. :param path: Path to the directory to serve. :param address: Optional address to listen on (defaults to ::) :param port: Optional port to listen on (defaults to TCP_GIT_PORT) """ # TODO(jelmer): Support git-daemon-export-ok and --export-all. backend = FileSystemBackend(path) server = TCPGitServer(backend, address, port) server.serve_forever() def web_daemon(path=".", address=None, port=None): """Run a daemon serving Git requests over HTTP. :param path: Path to the directory to serve :param address: Optional address to listen on (defaults to ::) :param port: Optional port to listen on (defaults to 80) """ from dulwich.web import ( make_wsgi_chain, make_server, WSGIRequestHandlerLogger, WSGIServerLogger) backend = FileSystemBackend(path) app = make_wsgi_chain(backend) server = make_server(address, port, app, handler_class=WSGIRequestHandlerLogger, server_class=WSGIServerLogger) server.serve_forever() def upload_pack(path=".", inf=None, outf=None): """Upload a pack file after negotiating its contents using smart protocol. :param path: Path to the repository :param inf: Input stream to communicate with client :param outf: Output stream to communicate with client """ if outf is None: outf = getattr(sys.stdout, 'buffer', sys.stdout) if inf is None: inf = getattr(sys.stdin, 'buffer', sys.stdin) path = os.path.expanduser(path) backend = FileSystemBackend(path) def send_fn(data): outf.write(data) outf.flush() proto = Protocol(inf.read, send_fn) handler = UploadPackHandler(backend, [path], proto) # FIXME: Catch exceptions and write a single-line summary to outf. handler.handle() return 0 def receive_pack(path=".", inf=None, outf=None): """Receive a pack file after negotiating its contents using smart protocol. :param path: Path to the repository :param inf: Input stream to communicate with client :param outf: Output stream to communicate with client """ if outf is None: outf = getattr(sys.stdout, 'buffer', sys.stdout) if inf is None: inf = getattr(sys.stdin, 'buffer', sys.stdin) path = os.path.expanduser(path) backend = FileSystemBackend(path) def send_fn(data): outf.write(data) outf.flush() proto = Protocol(inf.read, send_fn) handler = ReceivePackHandler(backend, [path], proto) # FIXME: Catch exceptions and write a single-line summary to outf. handler.handle() return 0 def _make_branch_ref(name): if getattr(name, 'encode', None): name = name.encode(DEFAULT_ENCODING) return b"refs/heads/" + name def _make_tag_ref(name): if getattr(name, 'encode', None): name = name.encode(DEFAULT_ENCODING) return b"refs/tags/" + name def branch_delete(repo, name): """Delete a branch. :param repo: Path to the repository :param name: Name of the branch """ with open_repo_closing(repo) as r: if isinstance(name, list): names = name else: names = [name] for name in names: del r.refs[_make_branch_ref(name)] def branch_create(repo, name, objectish=None, force=False): """Create a branch. :param repo: Path to the repository :param name: Name of the new branch :param objectish: Target object to point new branch at (defaults to HEAD) :param force: Force creation of branch, even if it already exists """ with open_repo_closing(repo) as r: if objectish is None: objectish = "HEAD" object = parse_object(r, objectish) refname = _make_branch_ref(name) ref_message = b"branch: Created from " + objectish.encode('utf-8') if force: r.refs.set_if_equals(refname, None, object.id, message=ref_message) else: if not r.refs.add_if_new(refname, object.id, message=ref_message): raise KeyError("Branch with name %s already exists." % name) def branch_list(repo): """List all branches. :param repo: Path to the repository """ with open_repo_closing(repo) as r: return r.refs.keys(base=b"refs/heads/") def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout, - errstream=default_bytes_err_stream, message=None, **kwargs): + errstream=default_bytes_err_stream, message=None, depth=None, **kwargs): """Fetch objects from a remote server. :param repo: Path to the repository :param remote_location: String identifying a remote server :param remote_name: Name for remote server :param outstream: Output stream (defaults to stdout) :param errstream: Error stream (defaults to stderr) :param message: Reflog message (defaults to b"fetch: from ") + :param depth: Depth to fetch at :return: Dictionary with refs on the remote """ if message is None: message = b'fetch: from ' + remote_location.encode("utf-8") with open_repo_closing(repo) as r: client, path = get_transport_and_path( remote_location, config=r.get_config_stack(), **kwargs) - fetch_result = client.fetch(path, r, progress=errstream.write) + fetch_result = client.fetch(path, r, progress=errstream.write, + depth=depth) stripped_refs = strip_peeled_refs(fetch_result.refs) branches = { n[len(b'refs/heads/'):]: v for (n, v) in stripped_refs.items() if n.startswith(b'refs/heads/')} r.refs.import_refs( b'refs/remotes/' + remote_name, branches, message=message) tags = { n[len(b'refs/tags/'):]: v for (n, v) in stripped_refs.items() if n.startswith(b'refs/tags/') and not n.endswith(ANNOTATED_TAG_SUFFIX)} r.refs.import_refs(b'refs/tags', tags, message=message) return fetch_result.refs def ls_remote(remote, config=None, **kwargs): """List the refs in a remote. :param remote: Remote repository location :param config: Configuration to use :return: Dictionary with remote refs """ if config is None: config = StackedConfig.default() client, host_path = get_transport_and_path(remote, config=config, **kwargs) return client.get_refs(host_path) def repack(repo): """Repack loose files in a repository. Currently this only packs loose objects. :param repo: Path to the repository """ with open_repo_closing(repo) as r: r.object_store.pack_loose_objects() def pack_objects(repo, object_ids, packf, idxf, delta_window_size=None): """Pack objects into a file. :param repo: Path to the repository :param object_ids: List of object ids to write :param packf: File-like object to write to :param idxf: File-like object to write to (can be None) """ with open_repo_closing(repo) as r: entries, data_sum = write_pack_objects( packf, r.object_store.iter_shas((oid, None) for oid in object_ids), delta_window_size=delta_window_size) if idxf is not None: entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()]) write_pack_index(idxf, entries, data_sum) def ls_tree(repo, treeish=b"HEAD", outstream=sys.stdout, recursive=False, name_only=False): """List contents of a tree. :param repo: Path to the repository :param tree_ish: Tree id to list :param outstream: Output stream (defaults to stdout) :param recursive: Whether to recursively list files :param name_only: Only print item name """ def list_tree(store, treeid, base): for (name, mode, sha) in store[treeid].iteritems(): if base: name = posixpath.join(base, name) if name_only: outstream.write(name + b"\n") else: outstream.write(pretty_format_tree_entry(name, mode, sha)) if stat.S_ISDIR(mode) and recursive: list_tree(store, sha, name) with open_repo_closing(repo) as r: tree = parse_tree(r, treeish) list_tree(r.object_store, tree.id, "") def remote_add(repo, name, url): """Add a remote. :param repo: Path to the repository :param name: Remote name :param url: Remote URL """ if not isinstance(name, bytes): name = name.encode(DEFAULT_ENCODING) if not isinstance(url, bytes): url = url.encode(DEFAULT_ENCODING) with open_repo_closing(repo) as r: c = r.get_config() section = (b'remote', name) if c.has_section(section): raise RemoteExists(section) c.set(section, b"url", url) c.write_to_path() def check_ignore(repo, paths, no_index=False): """Debug gitignore files. :param repo: Path to the repository :param paths: List of paths to check for :param no_index: Don't check index :return: List of ignored files """ with open_repo_closing(repo) as r: index = r.open_index() ignore_manager = IgnoreFilterManager.from_repo(r) for path in paths: if not no_index and path_to_tree_path(r.path, path) in index: continue if os.path.isabs(path): path = os.path.relpath(path, r.path) if ignore_manager.is_ignored(path): yield path def update_head(repo, target, detached=False, new_branch=None): """Update HEAD to point at a new branch/commit. Note that this does not actually update the working tree. :param repo: Path to the repository :param detach: Create a detached head :param target: Branch or committish to switch to :param new_branch: New branch to create """ with open_repo_closing(repo) as r: if new_branch is not None: to_set = _make_branch_ref(new_branch) else: to_set = b"HEAD" if detached: # TODO(jelmer): Provide some way so that the actual ref gets # updated rather than what it points to, so the delete isn't # necessary. del r.refs[to_set] r.refs[to_set] = parse_commit(r, target).id else: r.refs.set_symbolic_ref(to_set, parse_ref(r, target)) if new_branch is not None: r.refs.set_symbolic_ref(b"HEAD", to_set) def check_mailmap(repo, contact): """Check canonical name and email of contact. :param repo: Path to the repository :param contact: Contact name and/or email :return: Canonical contact data """ with open_repo_closing(repo) as r: from dulwich.mailmap import Mailmap import errno try: mailmap = Mailmap.from_path(os.path.join(r.path, '.mailmap')) except IOError as e: if e.errno != errno.ENOENT: raise mailmap = Mailmap() return mailmap.lookup(contact) def fsck(repo): """Check a repository. :param repo: A path to the repository :return: Iterator over errors/warnings """ with open_repo_closing(repo) as r: # TODO(jelmer): check pack files # TODO(jelmer): check graph # TODO(jelmer): check refs for sha in r.object_store: o = r.object_store[sha] try: o.check() except Exception as e: yield (sha, e) def stash_list(repo): """List all stashes in a repository.""" with open_repo_closing(repo) as r: from dulwich.stash import Stash stash = Stash.from_repo(r) return enumerate(list(stash.stashes())) def stash_push(repo): """Push a new stash onto the stack.""" with open_repo_closing(repo) as r: from dulwich.stash import Stash stash = Stash.from_repo(r) stash.push() def stash_pop(repo): """Pop a new stash from the stack.""" with open_repo_closing(repo) as r: from dulwich.stash import Stash stash = Stash.from_repo(r) stash.pop() def ls_files(repo): """List all files in an index.""" with open_repo_closing(repo) as r: return sorted(r.open_index()) def describe(repo): """Describe the repository version. :param projdir: git repository root :returns: a string description of the current git revision Examples: "gabcdefh", "v0.1" or "v0.1-5-gabcdefh". """ # Get the repository with open_repo_closing(repo) as r: # Get a list of all tags refs = r.get_refs() tags = {} for key, value in refs.items(): key = key.decode() obj = r.get_object(value) if u'tags' not in key: continue _, tag = key.rsplit(u'/', 1) try: commit = obj.object except AttributeError: continue else: commit = r.get_object(commit[1]) tags[tag] = [ datetime.datetime(*time.gmtime(commit.commit_time)[:6]), commit.id.decode('ascii'), ] sorted_tags = sorted(tags.items(), key=lambda tag: tag[1][0], reverse=True) # If there are no tags, return the current commit if len(sorted_tags) == 0: return 'g{}'.format(r[r.head()].id.decode('ascii')[:7]) # We're now 0 commits from the top commit_count = 0 # Get the latest commit latest_commit = r[r.head()] # Walk through all commits walker = r.get_walker() for entry in walker: # Check if tag commit_id = entry.commit.id.decode('ascii') for tag in sorted_tags: tag_name = tag[0] tag_commit = tag[1][1] if commit_id == tag_commit: if commit_count == 0: return tag_name else: return '{}-{}-g{}'.format( tag_name, commit_count, latest_commit.id.decode('ascii')[:7]) commit_count += 1 # Return plain commit if no parent tag can be found return 'g{}'.format(latest_commit.id.decode('ascii')[:7]) def get_object_by_path(repo, path, committish=None): """Get an object by path. :param repo: A path to the repository :param path: Path to look up :param committish: Commit to look up path in :return: A `ShaFile` object """ if committish is None: committish = "HEAD" # Get the repository with open_repo_closing(repo) as r: commit = parse_commit(repo, committish) base_tree = commit.tree if not isinstance(path, bytes): path = path.encode(commit.encoding or DEFAULT_ENCODING) (mode, sha) = tree_lookup_path( r.object_store.__getitem__, base_tree, path) return r[sha]