diff --git a/MANIFEST.in b/MANIFEST.in index 08ebc95..e7c46fc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include Makefile include requirements.txt +include requirements-swh.txt include version.txt diff --git a/PKG-INFO b/PKG-INFO index 08d4cb4..4925bc7 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.model -Version: 0.0.12 +Version: 0.0.13 Summary: Software Heritage data model Home-page: https://forge.softwareheritage.org/diffusion/DMOD/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/bin/git-revhash b/bin/git-revhash new file mode 100755 index 0000000..69d1d1c --- /dev/null +++ b/bin/git-revhash @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Use +# git-revhash 'tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\nparent 22c0fa5195a53f2e733ec75a9b6e9d1624a8b771\nauthor seanius 1138341044 +0000\ncommitter seanius 1138341044 +0000\n\nmaking dir structure...\n' # noqa +# output: 17a631d474f49bbebfdf3d885dcde470d7faafd7 + +echo -ne $* | git hash-object --stdin -t commit diff --git a/bin/swh-revhash b/bin/swh-revhash new file mode 100755 index 0000000..c7e2998 --- /dev/null +++ b/bin/swh-revhash @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +# Use: +# swh-revhash 'tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\nparent 22c0fa5195a53f2e733ec75a9b6e9d1624a8b771\nauthor seanius 1138341044 +0000\ncommitter seanius 1138341044 +0000\n\nmaking dir structure...\n' # noqa +# output: 17a631d474f49bbebfdf3d885dcde470d7faafd7 + +# To compare with git: +# git-revhash 'tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\nparent 22c0fa5195a53f2e733ec75a9b6e9d1624a8b771\nauthor seanius 1138341044 +0000\ncommitter seanius 1138341044 +0000\n\nmaking dir structure...\n' # noqa +# output: 17a631d474f49bbebfdf3d885dcde470d7faafd7 + + +import sys + +from swh.model import identifiers, hashutil + + +def revhash(revision_raw): + """Compute the revision hash. + + """ + if b'\\n' in revision_raw: # HACK: string have somehow their \n + # expanded to \\n + revision_raw = revision_raw.replace(b'\\n', b'\n') + + h = hashutil.hash_git_data(revision_raw, 'commit') + return identifiers.identifier_to_str(h) + + +if __name__ == '__main__': + revision_raw = sys.argv[1].encode('utf-8') + print(revhash(revision_raw)) diff --git a/requirements-swh.txt b/requirements-swh.txt new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py index 31541b4..a520dc0 100644 --- a/setup.py +++ b/setup.py @@ -1,31 +1,31 @@ from setuptools import setup def parse_requirements(): requirements = [] - with open('requirements.txt') as f: - for line in f.readlines(): - line = line.strip() - if not line or line.startswith('#'): - continue - requirements.append(line) - + for reqf in ('requirements.txt', 'requirements-swh.txt'): + with open(reqf) as f: + for line in f.readlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + requirements.append(line) return requirements setup( name='swh.model', description='Software Heritage data model', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DMOD/', packages=[ 'swh.model', 'swh.model.fields', 'swh.model.tests', 'swh.model.tests.fields', ], # packages's modules scripts=[], # scripts to package install_requires=parse_requirements(), setup_requires=['vcversioner'], vcversioner={}, include_package_data=True, ) diff --git a/swh.model.egg-info/PKG-INFO b/swh.model.egg-info/PKG-INFO index 08d4cb4..4925bc7 100644 --- a/swh.model.egg-info/PKG-INFO +++ b/swh.model.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.model -Version: 0.0.12 +Version: 0.0.13 Summary: Software Heritage data model Home-page: https://forge.softwareheritage.org/diffusion/DMOD/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh.model.egg-info/SOURCES.txt b/swh.model.egg-info/SOURCES.txt index 6d07beb..7927051 100644 --- a/swh.model.egg-info/SOURCES.txt +++ b/swh.model.egg-info/SOURCES.txt @@ -1,41 +1,44 @@ .gitignore AUTHORS LICENSE MANIFEST.in Makefile Makefile.local README-dev.md +requirements-swh.txt requirements.txt setup.py version.txt +bin/git-revhash +bin/swh-revhash debian/changelog debian/compat debian/control debian/copyright debian/rules debian/source/format swh.model.egg-info/PKG-INFO swh.model.egg-info/SOURCES.txt swh.model.egg-info/dependency_links.txt swh.model.egg-info/requires.txt swh.model.egg-info/top_level.txt swh/model/__init__.py swh/model/exceptions.py swh/model/git.py swh/model/hashutil.py swh/model/identifiers.py swh/model/validators.py swh/model/fields/__init__.py swh/model/fields/compound.py swh/model/fields/hashes.py swh/model/fields/simple.py swh/model/tests/__init__.py swh/model/tests/test_git.py swh/model/tests/test_git_slow.py swh/model/tests/test_hashutil.py swh/model/tests/test_identifiers.py swh/model/tests/test_validators.py swh/model/tests/fields/__init__.py swh/model/tests/fields/test_compound.py swh/model/tests/fields/test_hashes.py swh/model/tests/fields/test_simple.py \ No newline at end of file diff --git a/swh/model/git.py b/swh/model/git.py index 6cc7019..a3503cb 100644 --- a/swh/model/git.py +++ b/swh/model/git.py @@ -1,575 +1,575 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os from enum import Enum from swh.model import hashutil, identifiers ROOT_TREE_KEY = b'' class GitType(Enum): BLOB = b'blob' TREE = b'tree' EXEC = b'exec' LINK = b'link' COMM = b'commit' RELE = b'release' REFS = b'ref' class GitPerm(Enum): BLOB = b'100644' TREE = b'40000' EXEC = b'100755' LINK = b'120000' def _compute_directory_git_sha1(hashes): """Compute a directory git sha1 from hashes. Args: hashes: list of tree entries with keys: - sha1_git: the tree entry's sha1 - name: file or subdir's name - perms: the tree entry's sha1 permissions Returns: the binary sha1 of the dictionary's identifier Assumes: Every path exists in hashes. """ directory = { 'entries': [ { 'name': entry['name'], 'perms': int(entry['perms'].value, 8), 'target': entry['sha1_git'], 'type': 'dir' if entry['perms'] == GitPerm.TREE else 'file', } for entry in hashes ] } return hashutil.hash_to_bytes(identifiers.directory_identifier(directory)) def compute_directory_git_sha1(dirpath, hashes): """Compute a directory git sha1 for a dirpath. Args: dirpath: the directory's absolute path hashes: list of tree entries with keys: - sha1_git: the tree entry's sha1 - name: file or subdir's name - perms: the tree entry's sha1 permissions Returns: the binary sha1 of the dictionary's identifier Assumes: Every path exists in hashes. """ return _compute_directory_git_sha1(hashes[dirpath]) def compute_revision_sha1_git(revision): """Compute a revision sha1 git from its dict representation. Args: revision: Additional dictionary information needed to compute a synthetic revision. Following keys are expected: - author - date - committer - committer_date - message - type - directory: binary form of the tree hash Returns: revision sha1 in bytes # FIXME: beware, bytes output from storage api """ return hashutil.hash_to_bytes(identifiers.revision_identifier(revision)) def compute_release_sha1_git(release): """Compute a release sha1 git from its dict representation. Args: release: Additional dictionary information needed to compute a synthetic release. Following keys are expected: - name - message - date - author - revision: binary form of the sha1_git revision targeted by this Returns: release sha1 in bytes """ return hashutil.hash_to_bytes(identifiers.release_identifier(release)) def compute_link_metadata(linkpath): """Given a linkpath, compute the git metadata. Args: linkpath: absolute pathname of the link Returns: Dictionary of values: - data: link's content - length: link's content length - name: basename of the link - perms: git permission for link - type: git type for link - path: absolute path to the link on filesystem """ data = os.readlink(linkpath) link_metadata = hashutil.hash_data(data) link_metadata.update({ 'data': data, 'length': len(data), 'name': os.path.basename(linkpath), 'perms': GitPerm.LINK, 'type': GitType.BLOB, 'path': linkpath }) return link_metadata def compute_blob_metadata(filepath): """Given a filepath, compute the git metadata. Args: filepath: absolute pathname of the file. Returns: Dictionary of values: - name: basename of the file - perms: git permission for file - type: git type for file - path: absolute filepath on filesystem """ blob_metadata = hashutil.hash_path(filepath) perms = GitPerm.EXEC if os.access(filepath, os.X_OK) else GitPerm.BLOB blob_metadata.update({ 'name': os.path.basename(filepath), 'perms': perms, 'type': GitType.BLOB, 'path': filepath }) return blob_metadata def _compute_tree_metadata(dirname, hashes): """Given a dirname, compute the git metadata. Args: dirname: absolute pathname of the directory. hashes: list of tree dirname's entries with keys: - sha1_git: the tree entry's sha1 - name: file or subdir's name - perms: the tree entry's sha1 permissions Returns: Dictionary of values: - sha1_git: tree's sha1 git - name: basename of the directory - perms: git permission for directory - type: git type for directory - path: absolute path to directory on filesystem """ return { 'sha1_git': _compute_directory_git_sha1(hashes), 'name': os.path.basename(dirname), 'perms': GitPerm.TREE, 'type': GitType.TREE, 'path': dirname } def compute_tree_metadata(dirname, ls_hashes): """Given a dirname, compute the git metadata. Args: dirname: absolute pathname of the directory. ls_hashes: dictionary of path, hashes Returns: Dictionary of values: - sha1_git: tree's sha1 git - name: basename of the directory - perms: git permission for directory - type: git type for directory - path: absolute path to directory on filesystem """ return _compute_tree_metadata(dirname, ls_hashes[dirname]) def default_validation_dir(dirpath): """Default validation function. This is the equivalent of the identity function. Args: dirpath: Path to validate Returns: True """ return True -def __walk(rootdir, - dir_ok_fn=default_validation_dir, - remove_empty_folder=False): +def _walk(rootdir, + dir_ok_fn=default_validation_dir, + remove_empty_folder=False): """Walk the filesystem and yields a 3 tuples (dirpath, dirnames as set of absolute paths, filenames as set of abslute paths) Ignore files which won't pass the dir_ok_fn validation. If remove_empty_folder is True, remove and ignore any encountered empty folder. Args: - rootdir: starting walk root directory path - dir_ok_fn: validation function. if folder encountered are not ok, they are ignored. Default to default_validation_dir which does nothing. - remove_empty_folder: Flag to remove and ignore any encountered empty folders. Yields: 3 tuples dirpath, set of absolute children dirname paths, set of absolute filename paths. """ def basic_gen_dir(rootdir): for dp, dns, fns in os.walk(rootdir, topdown=False): yield (dp, set((os.path.join(dp, dn) for dn in dns)), set((os.path.join(dp, fn) for fn in fns))) if dir_ok_fn == default_validation_dir: if not remove_empty_folder: # os.walk yield from basic_gen_dir(rootdir) else: # os.walk + empty dir cleanup empty_folders = set() for dp, dns, fns in basic_gen_dir(rootdir): if not dns and not fns: empty_folders.add(dp) # need to remove it because folder of empty folder # is an empty folder!!! if os.path.islink(dp): os.remove(dp) else: os.rmdir(dp) parent = os.path.dirname(dp) # edge case about parent containing one empty # folder which become an empty one while not os.listdir(parent): empty_folders.add(parent) if os.path.islink(parent): os.remove(parent) else: os.rmdir(parent) parent = os.path.dirname(parent) continue yield (dp, dns - empty_folders, fns) else: def filtfn(dirnames): return set(filter(dir_ok_fn, dirnames)) gen_dir = ((dp, dns, fns) for dp, dns, fns in basic_gen_dir(rootdir) if dir_ok_fn(dp)) if not remove_empty_folder: # os.walk + filtering for dp, dns, fns in gen_dir: yield (dp, filtfn(dns), fns) else: # os.walk + filtering + empty dir cleanup empty_folders = set() for dp, dns, fns in gen_dir: dps = filtfn(dns) if not dps and not fns: empty_folders.add(dp) # need to remove it because folder of empty folder # is an empty folder!!! if os.path.islink(dp): os.remove(dp) else: os.rmdir(dp) parent = os.path.dirname(dp) # edge case about parent containing one empty # folder which become an empty one while not os.listdir(parent): empty_folders.add(parent) if os.path.islink(parent): os.remove(parent) else: os.rmdir(parent) parent = os.path.dirname(parent) continue yield dp, dps - empty_folders, fns def walk_and_compute_sha1_from_directory(rootdir, dir_ok_fn=default_validation_dir, with_root_tree=True, remove_empty_folder=False): """(Deprecated) TODO migrate the code to compute_hashes_from_directory. Compute git sha1 from directory rootdir. Args: - rootdir: Root directory from which beginning the git hash computation - dir_ok_fn: Filter function to filter directory according to rules defined in the function. By default, all folders are ok. Example override: dir_ok_fn = lambda dirpath: b'svn' not in dirpath - with_root_tree: Determine if we compute the upper root tree's checksums. As a default, we want it. One possible use case where this is not useful is the update (cf. `update_checksums_from`) Returns: Dictionary of entries with keys and as values a list of directory entries. Those are list of dictionary with keys: - 'perms' - 'type' - 'name' - 'sha1_git' - and specifically content: 'sha1', 'sha256', ... Note: One special key is ROOT_TREE_KEY to indicate the upper root of the directory (this is the revision's directory). Raises: Nothing If something is raised, this is a programmatic error. """ ls_hashes = {} all_links = set() if rootdir.endswith(b'/'): rootdir = rootdir.rstrip(b'/') - for dirpath, dirnames, filenames in __walk( + for dirpath, dirnames, filenames in _walk( rootdir, dir_ok_fn, remove_empty_folder): hashes = [] links = (file for file in filenames.union(dirnames) if os.path.islink(file)) for linkpath in links: all_links.add(linkpath) m_hashes = compute_link_metadata(linkpath) hashes.append(m_hashes) for filepath in (file for file in filenames if file not in all_links): m_hashes = compute_blob_metadata(filepath) hashes.append(m_hashes) ls_hashes[dirpath] = hashes dir_hashes = [] for fulldirname in (dir for dir in dirnames if dir not in all_links): tree_hash = _compute_tree_metadata(fulldirname, ls_hashes[fulldirname]) dir_hashes.append(tree_hash) ls_hashes[dirpath].extend(dir_hashes) if with_root_tree: # compute the current directory hashes root_hash = { 'sha1_git': _compute_directory_git_sha1(ls_hashes[rootdir]), 'path': rootdir, 'name': os.path.basename(rootdir), 'perms': GitPerm.TREE, 'type': GitType.TREE } ls_hashes[ROOT_TREE_KEY] = [root_hash] return ls_hashes def compute_hashes_from_directory(rootdir, dir_ok_fn=default_validation_dir, remove_empty_folder=False): """Compute git sha1 from directory rootdir. Args: - rootdir: Root directory from which beginning the git hash computation - dir_ok_fn: Filter function to filter directory according to rules defined in the function. By default, all folders are ok. Example override: dir_ok_fn = lambda dirpath: b'svn' not in dirpath Returns: Dictionary of entries with keys absolute path name. Path-name can be a file/link or directory. The associated value is a dictionary with: - checksums: the dictionary with the hashes for the link/file/dir Those are list of dictionary with keys: - 'perms' - 'type' - 'name' - 'sha1_git' - and specifically content: 'sha1', 'sha256', ... - children: Only for a directory, the set of children paths Note: One special key is the / which indicates the upper root of the directory (this is the revision's directory). Raises: Nothing If something is raised, this is a programmatic error. """ - def __get_dict_from_dirpath(_dict, path): + def _get_dict_from_dirpath(_dict, path): """Retrieve the default associated value for key path. """ return _dict.get(path, dict(children=set(), checksums=None)) - def __get_dict_from_filepath(_dict, path): + def _get_dict_from_filepath(_dict, path): """Retrieve the default associated value for key path. """ return _dict.get(path, dict(checksums=None)) ls_hashes = {} all_links = set() if rootdir.endswith(b'/'): rootdir = rootdir.rstrip(b'/') - for dirpath, dirnames, filenames in __walk( + for dirpath, dirnames, filenames in _walk( rootdir, dir_ok_fn, remove_empty_folder): - dir_entry = __get_dict_from_dirpath(ls_hashes, dirpath) + dir_entry = _get_dict_from_dirpath(ls_hashes, dirpath) children = dir_entry['children'] links = (file for file in filenames.union(dirnames) if os.path.islink(file)) for linkpath in links: all_links.add(linkpath) m_hashes = compute_link_metadata(linkpath) - d = __get_dict_from_filepath(ls_hashes, linkpath) + d = _get_dict_from_filepath(ls_hashes, linkpath) d['checksums'] = m_hashes ls_hashes[linkpath] = d children.add(linkpath) for filepath in (file for file in filenames if file not in all_links): m_hashes = compute_blob_metadata(filepath) - d = __get_dict_from_filepath(ls_hashes, filepath) + d = _get_dict_from_filepath(ls_hashes, filepath) d['checksums'] = m_hashes ls_hashes[filepath] = d children.add(filepath) for fulldirname in (dir for dir in dirnames if dir not in all_links): - d_hashes = __get_dict_from_dirpath(ls_hashes, fulldirname) + d_hashes = _get_dict_from_dirpath(ls_hashes, fulldirname) tree_hash = _compute_tree_metadata( fulldirname, (ls_hashes[p]['checksums'] for p in d_hashes['children']) ) - d = __get_dict_from_dirpath(ls_hashes, fulldirname) + d = _get_dict_from_dirpath(ls_hashes, fulldirname) d['checksums'] = tree_hash ls_hashes[fulldirname] = d children.add(fulldirname) dir_entry['children'] = children ls_hashes[dirpath] = dir_entry # compute the current directory hashes - d_hashes = __get_dict_from_dirpath(ls_hashes, rootdir) + d_hashes = _get_dict_from_dirpath(ls_hashes, rootdir) root_hash = { 'sha1_git': _compute_directory_git_sha1( (ls_hashes[p]['checksums'] for p in d_hashes['children']) ), 'path': rootdir, 'name': os.path.basename(rootdir), 'perms': GitPerm.TREE, 'type': GitType.TREE } d_hashes['checksums'] = root_hash ls_hashes[rootdir] = d_hashes return ls_hashes def children_hashes(children, objects): """Given a collection of children path, yield the corresponding hashes. Args: objects: objects hash as returned by git.compute_hashes_from_directory. children: collection of bytes path Yields: Dictionary hashes """ for p in children: c = objects.get(p) if c: h = c.get('checksums') if h: yield h def objects_per_type(filter_type, objects_per_path): """Given an object dictionary returned by `swh.model.git.compute_hashes_from_directory`, yields corresponding element type's hashes Args: filter_type: one of GitType enum objects_per_path: Yields: Elements of type filter_type's hashes """ for path, obj in objects_per_path.items(): o = obj['checksums'] if o['type'] == filter_type: if 'children' in obj: # for trees if obj['children']: o['children'] = children_hashes(obj['children'], objects_per_path) else: o['children'] = [] yield o diff --git a/swh/model/identifiers.py b/swh/model/identifiers.py index cf3b326..c53513a 100644 --- a/swh/model/identifiers.py +++ b/swh/model/identifiers.py @@ -1,465 +1,487 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import binascii import datetime from functools import lru_cache from .hashutil import hash_data, hash_git_data @lru_cache() def identifier_to_bytes(identifier): """Convert a text identifier to bytes. Args: identifier: an identifier, either a 40-char hexadecimal string or a bytes object of length 20 Returns: The length 20 bytestring corresponding to the given identifier Raises: ValueError if the identifier is of an unexpected type or length. """ if isinstance(identifier, bytes): if len(identifier) != 20: raise ValueError( 'Wrong length for bytes identifier %s, expected 20' % len(identifier)) return identifier if isinstance(identifier, str): if len(identifier) != 40: raise ValueError( 'Wrong length for str identifier %s, expected 40' % len(identifier)) return bytes.fromhex(identifier) raise ValueError('Wrong type for identifier %s, expected bytes or str' % identifier.__class__.__name__) @lru_cache() def identifier_to_str(identifier): """Convert an identifier to an hexadecimal string. Args: identifier: an identifier, either a 40-char hexadecimal string or a bytes object of length 20 Returns: The length 40 string corresponding to the given identifier, hex encoded Raises: ValueError if the identifier is of an unexpected type or length. """ if isinstance(identifier, str): if len(identifier) != 40: raise ValueError( 'Wrong length for str identifier %s, expected 40' % len(identifier)) return identifier if isinstance(identifier, bytes): if len(identifier) != 20: raise ValueError( 'Wrong length for bytes identifier %s, expected 20' % len(identifier)) return binascii.hexlify(identifier).decode() raise ValueError('Wrong type for identifier %s, expected bytes or str' % identifier.__class__.__name__) def content_identifier(content): """Return the intrinsic identifier for a content. A content's identifier is the sha1, sha1_git and sha256 checksums of its data. Args: content: a content conforming to the Software Heritage schema Returns: A dictionary with all the hashes for the data Raises: KeyError if the content doesn't have a data member. """ hashes = hash_data( content['data'], {'sha1', 'sha1_git', 'sha256'}, ) return hashes def _sort_key(entry): """The sorting key for tree entries""" if entry['type'] == 'dir': return entry['name'] + b'/' else: return entry['name'] @lru_cache() def _perms_to_bytes(perms): """Convert the perms value to its bytes representation""" oc = oct(perms)[2:] return oc.encode('ascii') def directory_identifier(directory): """Return the intrinsic identifier for a directory. A directory's identifier is the tree sha1 à la git of a directory listing, using the following algorithm, which is equivalent to the git algorithm for trees: 1. Entries of the directory are sorted using the name (or the name with '/' appended for directory entries) as key, in bytes order. 2. For each entry of the directory, the following bytes are output: - the octal representation of the permissions for the entry (stored in the 'perms' member), which is a representation of the entry type: b'100644' (int 33188) for files b'100755' (int 33261) for executable files b'120000' (int 40960) for symbolic links b'40000' (int 16384) for directories b'160000' (int 57344) for references to revisions - an ascii space (b'\x20') - the entry's name (as raw bytes), stored in the 'name' member - a null byte (b'\x00') - the 20 byte long identifier of the object pointed at by the entry, stored in the 'target' member: for files or executable files: their blob sha1_git for symbolic links: the blob sha1_git of a file containing the link destination for directories: their intrinsic identifier for revisions: their intrinsic identifier (Note that there is no separator between entries) """ components = [] for entry in sorted(directory['entries'], key=_sort_key): components.extend([ _perms_to_bytes(entry['perms']), b'\x20', entry['name'], b'\x00', identifier_to_bytes(entry['target']), ]) return identifier_to_str(hash_git_data(b''.join(components), 'tree')) def format_date(date): """Convert a date object into an UTC timestamp encoded as ascii bytes. Git stores timestamps as an integer number of seconds since the UNIX epoch. However, Software Heritage stores timestamps as an integer number of microseconds (postgres type "datetime with timezone"). Therefore, we print timestamps with no microseconds as integers, and - timestamps with microseconds as floating point values. + timestamps with microseconds as floating point values. We elide the + trailing zeroes from microsecond values, to "future-proof" our + representation if we ever need more precision in timestamps. """ - if isinstance(date, datetime.datetime): - if date.microsecond == 0: - date = int(date.timestamp()) - else: - date = date.timestamp() - return str(date).encode() + if not isinstance(date, dict): + raise ValueError('format_date only supports dicts, %r received' % date) + + seconds = date.get('seconds', 0) + microseconds = date.get('microseconds', 0) + if not microseconds: + return str(seconds).encode() else: - if date == int(date): - date = int(date) - return str(date).encode() + float_value = ('%d.%06d' % (seconds, microseconds)) + return float_value.rstrip('0').encode() @lru_cache() def format_offset(offset, negative_utc=None): """Convert an integer number of minutes into an offset representation. The offset representation is [+-]hhmm where: hh is the number of hours; mm is the number of minutes. A null offset is represented as +0000. """ if offset < 0 or offset == 0 and negative_utc: sign = '-' else: sign = '+' hours = abs(offset) // 60 minutes = abs(offset) % 60 t = '%s%02d%02d' % (sign, hours, minutes) return t.encode() def normalize_timestamp(time_representation): """Normalize a time representation for processing by Software Heritage This function supports a numeric timestamp (representing a number of seconds since the UNIX epoch, 1970-01-01 at 00:00 UTC), a datetime.datetime object (with timezone information), or a normalized Software Heritage time representation (idempotency). Args: time_representation: the representation of a timestamp Returns: a normalized dictionary with three keys - - timestamp: a number of seconds since the UNIX epoch (1970-01-01 at 00:00 - UTC) + - timestamp: a dict with two optional keys: + - seconds: the integral number of seconds since the UNIX epoch + - microseconds: the integral number of microseconds - offset: the timezone offset as a number of minutes relative to UTC - negative_utc: a boolean representing whether the offset is -0000 when offset = 0. """ if time_representation is None: return None negative_utc = False if isinstance(time_representation, dict): - timestamp = time_representation['timestamp'] + ts = time_representation['timestamp'] + if isinstance(ts, dict): + seconds = ts.get('seconds', 0) + microseconds = ts.get('microseconds', 0) + elif isinstance(ts, int): + seconds = ts + microseconds = 0 + else: + raise ValueError( + 'normalize_timestamp received non-integer timestamp member:' + ' %r' % ts) offset = time_representation['offset'] if 'negative_utc' in time_representation: negative_utc = time_representation['negative_utc'] elif isinstance(time_representation, datetime.datetime): - timestamp = time_representation.timestamp() + seconds = int(time_representation.timestamp()) + microseconds = time_representation.microsecond utcoffset = time_representation.utcoffset() if utcoffset is None: raise ValueError( 'normalize_timestamp received datetime without timezone: %s' % time_representation) # utcoffset is an integer number of minutes seconds_offset = utcoffset.total_seconds() offset = int(seconds_offset) // 60 - else: - timestamp = time_representation + elif isinstance(time_representation, int): + seconds = time_representation + microseconds = 0 offset = 0 + else: + raise ValueError( + 'normalize_timestamp received non-integer timestamp:' + ' %r' % time_representation) return { - 'timestamp': timestamp, + 'timestamp': { + 'seconds': seconds, + 'microseconds': microseconds, + }, 'offset': offset, 'negative_utc': negative_utc, } def format_author(author): """Format the specification of an author. An author is either a byte string (passed unchanged), or a dict with three keys, fullname, name and email. If the fullname exists, return it; if it doesn't, we construct a fullname using the following heuristics: if the name value is None, we return the email in angle brackets, else, we return the name, a space, and the email in angle brackets. """ if isinstance(author, bytes) or author is None: return author if 'fullname' in author: return author['fullname'] ret = [] if author['name'] is not None: ret.append(author['name']) if author['email'] is not None: ret.append(b''.join([b'<', author['email'], b'>'])) return b' '.join(ret) def format_author_line(header, author, date_offset): """Format a an author line according to git standards. An author line has three components: - a header, describing the type of author (author, committer, tagger) - a name and email, which is an arbitrary bytestring - optionally, a timestamp with UTC offset specification The author line is formatted thus: `header` `name and email`[ `timestamp` `utc_offset`] The timestamp is encoded as a (decimal) number of seconds since the UNIX epoch (1970-01-01 at 00:00 UTC). As an extension to the git format, we support fractional timestamps, using a dot as the separator for the decimal part. The utc offset is a number of minutes encoded as '[+-]HHMM'. Note some tools can pass a negative offset corresponding to the UTC timezone ('-0000'), which is valid and is encoded as such. For convenience, this function returns the whole line with its trailing newline. Args: header: the header of the author line (one of 'author', 'committer', 'tagger') author: an author specification (dict with two bytes values: name and email, or byte value) date_offset: a normalized date/time representation as returned by `normalize_timestamp`. Returns: the newline-terminated byte string containing the author line """ ret = [header.encode(), b' ', format_author(author)] date_offset = normalize_timestamp(date_offset) if date_offset is not None: date_f = format_date(date_offset['timestamp']) offset_f = format_offset(date_offset['offset'], date_offset['negative_utc']) ret.extend([b' ', date_f, b' ', offset_f]) ret.append(b'\n') return b''.join(ret) def revision_identifier(revision): """Return the intrinsic identifier for a revision. The fields used for the revision identifier computation are: - directory - parents - author - author_date - committer - committer_date - metadata -> extra_headers - message A revision's identifier is the 'git'-checksum of a commit manifest constructed as follows (newlines are a single ASCII newline character): ``` tree [for each parent in parents] parent [end for each parents] author committer [for each key, value in extra_headers] [end for each extra_headers] ``` The directory identifier is the ascii representation of its hexadecimal encoding. Author and committer are formatted with the `format_author` function. Dates are formatted with the `format_date_offset` function. Extra headers are an ordered list of [key, value] pairs. Keys are strings and get encoded to utf-8 for identifier computation. Values are either byte strings, unicode strings (that get encoded to utf-8), or integers (that get encoded to their utf-8 decimal representation). Multiline extra header values are escaped by indenting the continuation lines with one ascii space. If the message is None, the manifest ends with the last header. Else, the message is appended to the headers after an empty line. The checksum of the full manifest is computed using the 'commit' git object type. """ components = [ b'tree ', identifier_to_str(revision['directory']).encode(), b'\n', ] for parent in revision['parents']: if parent: components.extend([ b'parent ', identifier_to_str(parent).encode(), b'\n', ]) components.extend([ format_author_line('author', revision['author'], revision['date']), format_author_line('committer', revision['committer'], revision['committer_date']), ]) # Handle extra headers metadata = revision.get('metadata') if not metadata: metadata = {} for key, value in metadata.get('extra_headers', []): # Integer values: decimal representation if isinstance(value, int): value = str(value).encode('utf-8') # Unicode string values: utf-8 encoding if isinstance(value, str): value = value.encode('utf-8') # multi-line values: indent continuation lines if b'\n' in value: value_chunks = value.split(b'\n') value = b'\n '.join(value_chunks) # encode the key to utf-8 components.extend([key.encode('utf-8'), b' ', value, b'\n']) if revision['message'] is not None: components.extend([b'\n', revision['message']]) commit_raw = b''.join(components) return identifier_to_str(hash_git_data(commit_raw, 'commit')) def target_type_to_git(target_type): """Convert a software heritage target type to a git object type""" return { 'content': b'blob', 'directory': b'tree', 'revision': b'commit', 'release': b'tag', }[target_type] def release_identifier(release): """Return the intrinsic identifier for a release.""" components = [ b'object ', identifier_to_str(release['target']).encode(), b'\n', b'type ', target_type_to_git(release['target_type']), b'\n', b'tag ', release['name'], b'\n', ] if 'author' in release and release['author']: components.append( format_author_line('tagger', release['author'], release['date']) ) if release['message'] is not None: components.extend([b'\n', release['message']]) return identifier_to_str(hash_git_data(b''.join(components), 'tag')) diff --git a/swh/model/tests/test_git.py b/swh/model/tests/test_git.py index f58057a..b1eac8c 100644 --- a/swh/model/tests/test_git.py +++ b/swh/model/tests/test_git.py @@ -1,687 +1,687 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import shutil import subprocess import tempfile import unittest from nose.plugins.attrib import attr from nose.tools import istest from swh.model import git class GitHashlib(unittest.TestCase): def setUp(self): self.tree_data = b''.join([b'40000 barfoo\0', bytes.fromhex('c3020f6bf135a38c6df' '3afeb5fb38232c5e07087'), b'100644 blah\0', bytes.fromhex('63756ef0df5e4f10b6efa' '33cfe5c758749615f20'), b'100644 hello\0', bytes.fromhex('907b308167f0880fb2a' '5c0e1614bb0c7620f9dc3')]) self.commit_data = """tree 1c61f7259dcb770f46b194d941df4f08ff0a3970 author Antoine R. Dumont (@ardumont) 1444054085 +0200 committer Antoine R. Dumont (@ardumont) 1444054085 +0200 initial """.encode('utf-8') # NOQA self.tag_data = """object 24d012aaec0bc5a4d2f62c56399053d6cc72a241 type commit tag 0.0.1 tagger Antoine R. Dumont (@ardumont) 1444225145 +0200 blah """.encode('utf-8') # NOQA self.checksums = { 'tree_sha1_git': bytes.fromhex('ac212302c45eada382b27bfda795db' '121dacdb1c'), 'commit_sha1_git': bytes.fromhex('e960570b2e6e2798fa4cfb9af2c399' 'd629189653'), 'tag_sha1_git': bytes.fromhex('bc2b99ba469987bcf1272c189ed534' 'e9e959f120'), } @istest def compute_directory_git_sha1(self): # given dirpath = 'some-dir-path' hashes = { dirpath: [{'perms': git.GitPerm.TREE, 'type': git.GitType.TREE, 'name': b'barfoo', 'sha1_git': bytes.fromhex('c3020f6bf135a38c6df' '3afeb5fb38232c5e07087')}, {'perms': git.GitPerm.BLOB, 'type': git.GitType.BLOB, 'name': b'hello', 'sha1_git': bytes.fromhex('907b308167f0880fb2a' '5c0e1614bb0c7620f9dc3')}, {'perms': git.GitPerm.BLOB, 'type': git.GitType.BLOB, 'name': b'blah', 'sha1_git': bytes.fromhex('63756ef0df5e4f10b6efa' '33cfe5c758749615f20')}] } # when checksum = git.compute_directory_git_sha1(dirpath, hashes) # then self.assertEqual(checksum, self.checksums['tree_sha1_git']) @istest def compute_revision_sha1_git(self): # given tree_hash = bytes.fromhex('1c61f7259dcb770f46b194d941df4f08ff0a3970') revision = { 'author': { 'name': b'Antoine R. Dumont (@ardumont)', 'email': b'antoine.romain.dumont@gmail.com', }, 'date': { 'timestamp': 1444054085, 'offset': 120, }, 'committer': { 'name': b'Antoine R. Dumont (@ardumont)', 'email': b'antoine.romain.dumont@gmail.com', }, 'committer_date': { 'timestamp': 1444054085, 'offset': 120, }, 'message': b'initial\n', 'type': 'tar', 'directory': tree_hash, 'parents': [], } # when checksum = git.compute_revision_sha1_git(revision) # then self.assertEqual(checksum, self.checksums['commit_sha1_git']) @istest def compute_release_sha1_git(self): # given revision_hash = bytes.fromhex('24d012aaec0bc5a4d2f62c56399053' 'd6cc72a241') release = { 'name': b'0.0.1', 'author': { 'name': b'Antoine R. Dumont (@ardumont)', 'email': b'antoine.romain.dumont@gmail.com', }, 'date': { 'timestamp': 1444225145, 'offset': 120, }, 'message': b'blah\n', 'target_type': 'revision', 'target': revision_hash, } # when checksum = git.compute_release_sha1_git(release) # then self.assertEqual(checksum, self.checksums['tag_sha1_git']) @attr('fs') class GitHashWalkArborescenceTree: """Root class to ease walk and git hash testing without side-effecty problems. """ def setUp(self): super().setUp() self.tmp_root_path = tempfile.mkdtemp().encode('utf-8') self.maxDiff = None start_path = os.path.dirname(__file__).encode('utf-8') sample_folder = os.path.join(start_path, b'../../../..', b'swh-storage-testdata', b'dir-folders', b'sample-folder.tgz') self.root_path = os.path.join(self.tmp_root_path, b'sample-folder') # uncompress the sample folder subprocess.check_output( ['tar', 'xvf', sample_folder, '-C', self.tmp_root_path]) def tearDown(self): if os.path.exists(self.tmp_root_path): shutil.rmtree(self.tmp_root_path) class GitHashFromScratch(GitHashWalkArborescenceTree, unittest.TestCase): """Test the main `walk_and_compute_sha1_from_directory` algorithm that scans and compute the disk for checksums. """ @istest def walk_and_compute_sha1_from_directory(self): # make a temporary arborescence tree to hash without ignoring anything # same as previous behavior walk0 = git.walk_and_compute_sha1_from_directory(self.tmp_root_path) keys0 = list(walk0.keys()) path_excluded = os.path.join(self.tmp_root_path, b'sample-folder', b'foo') self.assertTrue(path_excluded in keys0) # it is not excluded here # make the same temporary arborescence tree to hash with ignoring one # folder foo walk1 = git.walk_and_compute_sha1_from_directory( self.tmp_root_path, dir_ok_fn=lambda dirpath: b'sample-folder/foo' not in dirpath) keys1 = list(walk1.keys()) self.assertTrue(path_excluded not in keys1) # remove the keys that can't be the same (due to hash definition) # Those are the top level folders keys_diff = [self.tmp_root_path, os.path.join(self.tmp_root_path, b'sample-folder'), git.ROOT_TREE_KEY] for k in keys_diff: self.assertNotEquals(walk0[k], walk1[k]) # The remaining keys (bottom path) should have exactly the same hashes # as before keys = set(keys1) - set(keys_diff) actual_walk1 = {} for k in keys: self.assertEquals(walk0[k], walk1[k]) actual_walk1[k] = walk1[k] expected_checksums = { os.path.join(self.tmp_root_path, b'sample-folder/empty-folder'): [], # noqa os.path.join(self.tmp_root_path, b'sample-folder/bar/barfoo'): [{ # noqa 'type': git.GitType.BLOB, # noqa 'length': 72, 'sha256': b'=\xb5\xae\x16\x80U\xbc\xd9:M\x08(]\xc9\x9f\xfe\xe2\x883\x03\xb2?\xac^\xab\x85\x02s\xa8\xeaUF', # noqa 'name': b'another-quote.org', # noqa 'path': os.path.join(self.tmp_root_path, b'sample-folder/bar/barfoo/another-quote.org'), # noqa 'perms': git.GitPerm.BLOB, # noqa 'sha1': b'\x90\xa6\x13\x8b\xa5\x99\x15&\x1e\x17\x99H8j\xa1\xcc*\xa9"\n', # noqa 'sha1_git': b'\x136\x93\xb1%\xba\xd2\xb4\xac1\x855\xb8I\x01\xeb\xb1\xf6\xb68'}], # noqa os.path.join(self.tmp_root_path, b'sample-folder/bar'): [{ # noqa 'type': git.GitType.TREE, # noqa 'perms': git.GitPerm.TREE, # noqa 'name': b'barfoo', # noqa 'path': os.path.join(self.tmp_root_path, b'sample-folder/bar/barfoo'), # noqa 'sha1_git': b'\xc3\x02\x0fk\xf15\xa3\x8cm\xf3\xaf\xeb_\xb3\x822\xc5\xe0p\x87'}]} # noqa self.assertEquals(actual_walk1, expected_checksums) @istest def walk_and_compute_sha1_from_directory_without_root_tree(self): # compute the full checksums expected_hashes = git.walk_and_compute_sha1_from_directory( self.tmp_root_path) # except for the key on that round actual_hashes = git.walk_and_compute_sha1_from_directory( self.tmp_root_path, with_root_tree=False) # then, removing the root tree hash from the first round del expected_hashes[git.ROOT_TREE_KEY] # should give us the same checksums as the second round self.assertEquals(actual_hashes, expected_hashes) class WithSampleFolderChecksums: def setUp(self): super().setUp() self.rootkey = b'/tmp/tmp7w3oi_j8' self.objects = { b'/tmp/tmp7w3oi_j8': { 'children': {b'/tmp/tmp7w3oi_j8/sample-folder'}, 'checksums': { 'type': git.GitType.TREE, 'name': b'tmp7w3oi_j8', 'sha1_git': b'\xa7A\xfcM\x96\x8c{\x8e<\x94\xff\x86\xe7\x04\x80\xc5\xc7\xe5r\xa9', # noqa 'path': b'/tmp/tmp7w3oi_j8', 'perms': git.GitPerm.TREE }, }, b'/tmp/tmp7w3oi_j8/sample-folder': { 'children': { b'/tmp/tmp7w3oi_j8/sample-folder/empty-folder', b'/tmp/tmp7w3oi_j8/sample-folder/link-to-binary', b'/tmp/tmp7w3oi_j8/sample-folder/link-to-another-quote', b'/tmp/tmp7w3oi_j8/sample-folder/link-to-foo', b'/tmp/tmp7w3oi_j8/sample-folder/some-binary', b'/tmp/tmp7w3oi_j8/sample-folder/bar', b'/tmp/tmp7w3oi_j8/sample-folder/foo', }, 'checksums': { 'type': git.GitType.TREE, 'name': b'sample-folder', 'sha1_git': b'\xe8\xb0\xf1Fj\xf8`\x8c\x8a?\xb9\x87\x9d\xb1r\xb8\x87\xe8\x07Y', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder', 'perms': git.GitPerm.TREE} }, b'/tmp/tmp7w3oi_j8/sample-folder/empty-folder': { 'children': {}, 'checksums': { 'type': git.GitType.TREE, 'name': b'empty-folder', 'sha1_git': b'K\x82]\xc6B\xcbn\xb9\xa0`\xe5K\xf8\xd6\x92\x88\xfb\xeeI\x04', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/empty-folder', 'perms': git.GitPerm.TREE } }, b'/tmp/tmp7w3oi_j8/sample-folder/link-to-binary': { 'checksums': { 'name': b'link-to-binary', 'sha1': b'\xd0$\x87\x14\x94\x8b:H\xa2T8#*o\x99\xf01\x8fY\xf1', # noqa 'data': b'some-binary', 'sha1_git': b'\xe8kE\xe58\xd9\xb6\x88\x8c\x96\x9c\x89\xfb\xd2*\x85\xaa\x0e\x03f', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/link-to-binary', 'sha256': b'\x14\x12n\x97\xd8?}&\x1cZh\x89\xce\xe76\x19w\x0f\xf0\x9e@\xc5I\x86\x85\xab\xa7E\xbe\x88.\xff', # noqa 'perms': git.GitPerm.LINK, 'type': git.GitType.BLOB, 'length': 11 } }, b'/tmp/tmp7w3oi_j8/sample-folder/link-to-another-quote': { 'checksums': { 'name': b'link-to-another-quote', 'sha1': b'\xcb\xee\xd1^yY\x9c\x90\xdes\x83\xf4 \xfe\xd7\xac\xb4\x8e\xa1q', # noqa 'data': b'bar/barfoo/another-quote.org', 'sha1_git': b'}\\\x08\x11\x1e!\xc8\xa9\xf7\x15@\x93\x99\x98U\x16\x837_\xad', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/link-to-another-quote', # noqa 'sha256': b'\xe6\xe1}\x07\x93\xaau\n\x04@\xeb\x9a\xd5\xb8\x0b%\x80vc~\xf0\xfbh\xf3\xac.Y\xe4\xb9\xac;\xa6', # noqa 'perms': git.GitPerm.LINK, 'type': git.GitType.BLOB, 'length': 28 } }, b'/tmp/tmp7w3oi_j8/sample-folder/link-to-foo': { 'checksums': { 'name': b'link-to-foo', 'sha1': b'\x0b\xee\xc7\xb5\xea?\x0f\xdb\xc9]\r\xd4\x7f<[\xc2u\xda\x8a3', # noqa 'data': b'foo', 'sha1_git': b'\x19\x10(\x15f=#\xf8\xb7ZG\xe7\xa0\x19e\xdc\xdc\x96F\x8c', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/link-to-foo', 'sha256': b',&\xb4kh\xff\xc6\x8f\xf9\x9bE<\x1d0A4\x13B-pd\x83\xbf\xa0\xf9\x8a^\x88bf\xe7\xae', # noqa 'perms': git.GitPerm.LINK, 'type': git.GitType.BLOB, 'length': 3 } }, b'/tmp/tmp7w3oi_j8/sample-folder/some-binary': { 'checksums': { 'name': b'some-binary', 'sha1': b'\x0b\xbc\x12\xd7\xf4\xa2\xa1[\x14=\xa8F\x17\xd9\\\xb2#\xc9\xb2<', # noqa 'sha1_git': b'hv\x95y\xc3\xea\xad\xbeUSy\xb9\xc3S\x8ef(\xba\xe1\xeb', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/some-binary', 'sha256': b'\xba\xc6P\xd3Jv8\xbb\n\xebSBdm$\xe3\xb9\xadkD\xc9\xb3\x83b\x1f\xaaH+\x99\n6}', # noqa 'perms': git.GitPerm.EXEC, 'type': git.GitType.BLOB, 'length': 5} }, b'/tmp/tmp7w3oi_j8/sample-folder/bar': { 'children': {b'/tmp/tmp7w3oi_j8/sample-folder/bar/barfoo'}, 'checksums': {'type': git.GitType.TREE, 'name': b'bar', 'sha1_git': b'<\x1fW\x83\x94\xf4b?t\xa0\xba\x7f\xe7ar\x9fY\xfcn\xc4', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/bar', 'perms': git.GitPerm.TREE}, }, b'/tmp/tmp7w3oi_j8/sample-folder/bar/barfoo': { 'children': {b'/tmp/tmp7w3oi_j8/sample-folder/bar/barfoo/another-quote.org'}, # noqa 'checksums': {'type': git.GitType.TREE, 'name': b'barfoo', 'sha1_git': b'\xc3\x02\x0fk\xf15\xa3\x8cm\xf3\xaf\xeb_\xb3\x822\xc5\xe0p\x87', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/bar/barfoo', # noqa 'perms': git.GitPerm.TREE}, }, b'/tmp/tmp7w3oi_j8/sample-folder/bar/barfoo/another-quote.org': { 'checksums': {'name': b'another-quote.org', 'sha1': b'\x90\xa6\x13\x8b\xa5\x99\x15&\x1e\x17\x99H8j\xa1\xcc*\xa9"\n', # noqa 'sha1_git': b'\x136\x93\xb1%\xba\xd2\xb4\xac1\x855\xb8I\x01\xeb\xb1\xf6\xb68', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/bar/barfoo/another-quote.org', # noqa 'sha256': b'=\xb5\xae\x16\x80U\xbc\xd9:M\x08(]\xc9\x9f\xfe\xe2\x883\x03\xb2?\xac^\xab\x85\x02s\xa8\xeaUF', # noqa 'perms': git.GitPerm.BLOB, 'type': git.GitType.BLOB, 'length': 72} }, b'/tmp/tmp7w3oi_j8/sample-folder/foo': { 'children': { b'/tmp/tmp7w3oi_j8/sample-folder/foo/barfoo', b'/tmp/tmp7w3oi_j8/sample-folder/foo/rel-link-to-barfoo', b'/tmp/tmp7w3oi_j8/sample-folder/foo/quotes.md', }, 'checksums': {'type': git.GitType.TREE, 'name': b'foo', 'sha1_git': b'+A\xc4\x0f\r\x1f\xbf\xfc\xba\x12I}\xb7\x1f\xba\x83\xfc\xca\x96\xe5', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/foo', 'perms': git.GitPerm.TREE} }, b'/tmp/tmp7w3oi_j8/sample-folder/foo/barfoo': { 'checksums': {'name': b'barfoo', 'sha1': b'\x90W\xeem\x01bPn\x01\xc4\xd9\xd5E\x9az\xdd\x1f\xed\xac7', # noqa 'data': b'bar/barfoo', 'sha1_git': b'\x81\x85\xdf\xb2\xc0\xc2\xc5\x97\xd1ou\xa8\xa0\xc3vhV|=~', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/foo/barfoo', # noqa 'sha256': b')\xad?W%2\x1b\x94\x032\xc7\x8e@6\x01\xaf\xffa\xda\xea\x85\xe9\xc8\x0bJpc\xb6\x88~\xadh', # noqa 'perms': git.GitPerm.LINK, 'type': git.GitType.BLOB, 'length': 10} }, b'/tmp/tmp7w3oi_j8/sample-folder/foo/rel-link-to-barfoo': { 'checksums': {'name': b'rel-link-to-barfoo', 'sha1': b'\xdcQ"\x1d0\x8f:\xeb\'T\xdbH9\x1b\x85h|(i\xf4', # noqa 'data': b'../bar/barfoo', 'sha1_git': b'\xac\xac2m\xddc\xb0\xbcp\x84\x06Y\xd4\xacCa\x94\x84\xe6\x9f', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/foo/rel-link-to-barfoo', # noqa 'sha256': b'\x80\x07\xd2\r\xb2\xaf@C_B\xdd\xefK\x8a\xd7k\x80\xad\xbe\xc2k$\x9f\xdf\x04s5?\x8d\x99\xdf\x08', # noqa 'perms': git.GitPerm.LINK, 'type': git.GitType.BLOB, 'length': 13} }, b'/tmp/tmp7w3oi_j8/sample-folder/foo/quotes.md': { 'checksums': {'name': b'quotes.md', 'sha1': b'\x1b\xf0\xbbr\x1a\xc9,\x18\xa1\x9b\x13\xc0\xeb=t\x1c\xbf\xad\xeb\xfc', # noqa 'sha1_git': b'|LW\xba\x9f\xf4\x96\xad\x17\x9b\x8fe\xb1\xd2\x86\xed\xbd\xa3L\x9a', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/foo/quotes.md', # noqa 'sha256': b'\xca\xca\x94*\xed\xa7\xb3\x08\x85\x9e\xb5o\x90\x9e\xc9m\x07\xa4\x99I\x16\x90\xc4S\xf7;\x98\x00\xa9;\x16Y', # noqa 'perms': git.GitPerm.BLOB, 'type': git.GitType.BLOB, 'length': 66} }, } class TestObjectsPerType(WithSampleFolderChecksums, unittest.TestCase): @istest def objects_per_type_blob(self): # given expected_blobs = [ { 'name': b'another-quote.org', 'sha1': b'\x90\xa6\x13\x8b\xa5\x99\x15&\x1e\x17\x99H8j\xa1\xcc*\xa9"\n', # noqa 'sha1_git': b'\x136\x93\xb1%\xba\xd2\xb4\xac1\x855\xb8I\x01\xeb\xb1\xf6\xb68', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/bar/barfoo/another-quote.org', # noqa 'sha256': b'=\xb5\xae\x16\x80U\xbc\xd9:M\x08(]\xc9\x9f\xfe\xe2\x883\x03\xb2?\xac^\xab\x85\x02s\xa8\xeaUF', # noqa 'perms': git.GitPerm.BLOB, 'type': git.GitType.BLOB, 'length': 72 }, { 'name': b'link-to-binary', 'sha1': b'\xd0$\x87\x14\x94\x8b:H\xa2T8#*o\x99\xf01\x8fY\xf1', 'data': b'some-binary', 'sha1_git': b'\xe8kE\xe58\xd9\xb6\x88\x8c\x96\x9c\x89\xfb\xd2*\x85\xaa\x0e\x03f', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/link-to-binary', 'sha256': b'\x14\x12n\x97\xd8?}&\x1cZh\x89\xce\xe76\x19w\x0f\xf0\x9e@\xc5I\x86\x85\xab\xa7E\xbe\x88.\xff', # noqa 'perms': git.GitPerm.LINK, 'type': git.GitType.BLOB, 'length': 11 }, { 'name': b'link-to-another-quote', 'sha1': b'\xcb\xee\xd1^yY\x9c\x90\xdes\x83\xf4 \xfe\xd7\xac\xb4\x8e\xa1q', # noqa 'data': b'bar/barfoo/another-quote.org', 'sha1_git': b'}\\\x08\x11\x1e!\xc8\xa9\xf7\x15@\x93\x99\x98U\x16\x837_\xad', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/link-to-another-quote', # noqa 'sha256': b'\xe6\xe1}\x07\x93\xaau\n\x04@\xeb\x9a\xd5\xb8\x0b%\x80vc~\xf0\xfbh\xf3\xac.Y\xe4\xb9\xac;\xa6', # noqa 'perms': git.GitPerm.LINK, 'type': git.GitType.BLOB, 'length': 28 }, { 'name': b'link-to-foo', 'sha1': b'\x0b\xee\xc7\xb5\xea?\x0f\xdb\xc9]\r\xd4\x7f<[\xc2u\xda\x8a3', # noqa 'data': b'foo', 'sha1_git': b'\x19\x10(\x15f=#\xf8\xb7ZG\xe7\xa0\x19e\xdc\xdc\x96F\x8c', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/link-to-foo', 'sha256': b',&\xb4kh\xff\xc6\x8f\xf9\x9bE<\x1d0A4\x13B-pd\x83\xbf\xa0\xf9\x8a^\x88bf\xe7\xae', # noqa 'perms': git.GitPerm.LINK, 'type': git.GitType.BLOB, 'length': 3 }, { 'name': b'some-binary', 'sha1': b'\x0b\xbc\x12\xd7\xf4\xa2\xa1[\x14=\xa8F\x17\xd9\\\xb2#\xc9\xb2<', # noqa 'sha1_git': b'hv\x95y\xc3\xea\xad\xbeUSy\xb9\xc3S\x8ef(\xba\xe1\xeb', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/some-binary', 'sha256': b'\xba\xc6P\xd3Jv8\xbb\n\xebSBdm$\xe3\xb9\xadkD\xc9\xb3\x83b\x1f\xaaH+\x99\n6}', # noqa 'perms': git.GitPerm.EXEC, 'type': git.GitType.BLOB, 'length': 5 }, { 'name': b'barfoo', 'sha1': b'\x90W\xeem\x01bPn\x01\xc4\xd9\xd5E\x9az\xdd\x1f\xed\xac7', # noqa 'data': b'bar/barfoo', 'sha1_git': b'\x81\x85\xdf\xb2\xc0\xc2\xc5\x97\xd1ou\xa8\xa0\xc3vhV|=~', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/foo/barfoo', 'sha256': b')\xad?W%2\x1b\x94\x032\xc7\x8e@6\x01\xaf\xffa\xda\xea\x85\xe9\xc8\x0bJpc\xb6\x88~\xadh', # noqa 'perms': git.GitPerm.LINK, 'type': git.GitType.BLOB, 'length': 10 }, { 'name': b'rel-link-to-barfoo', 'sha1': b'\xdcQ"\x1d0\x8f:\xeb\'T\xdbH9\x1b\x85h|(i\xf4', 'data': b'../bar/barfoo', 'sha1_git': b'\xac\xac2m\xddc\xb0\xbcp\x84\x06Y\xd4\xacCa\x94\x84\xe6\x9f', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/foo/rel-link-to-barfoo', # noqa 'sha256': b'\x80\x07\xd2\r\xb2\xaf@C_B\xdd\xefK\x8a\xd7k\x80\xad\xbe\xc2k$\x9f\xdf\x04s5?\x8d\x99\xdf\x08', # noqa 'perms': git.GitPerm.LINK, 'type': git.GitType.BLOB, 'length': 13 }, { 'name': b'quotes.md', 'sha1': b'\x1b\xf0\xbbr\x1a\xc9,\x18\xa1\x9b\x13\xc0\xeb=t\x1c\xbf\xad\xeb\xfc', # noqa 'sha1_git': b'|LW\xba\x9f\xf4\x96\xad\x17\x9b\x8fe\xb1\xd2\x86\xed\xbd\xa3L\x9a', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/foo/quotes.md', 'sha256': b'\xca\xca\x94*\xed\xa7\xb3\x08\x85\x9e\xb5o\x90\x9e\xc9m\x07\xa4\x99I\x16\x90\xc4S\xf7;\x98\x00\xa9;\x16Y', # noqa 'perms': git.GitPerm.BLOB, 'type': git.GitType.BLOB, 'length': 66 }, ] expected_sha1_blobs = set( ((c['sha1_git'], git.GitType.BLOB) for c in expected_blobs)) # when actual_sha1_blobs = set( ((c['sha1_git'], c['type']) for c in git.objects_per_type(git.GitType.BLOB, self.objects))) # then self.assertEqual(actual_sha1_blobs, expected_sha1_blobs) @istest def objects_per_type_tree(self): - def __children_hashes(path, objects=self.objects): + def _children_hashes(path, objects=self.objects): return set((c['sha1_git'] for c in git.children_hashes( objects[path]['children'], objects))) expected_trees = [ { 'type': git.GitType.TREE, 'name': b'tmp7w3oi_j8', 'sha1_git': b'\xa7A\xfcM\x96\x8c{\x8e<\x94\xff\x86\xe7\x04\x80\xc5\xc7\xe5r\xa9', # noqa 'path': b'/tmp/tmp7w3oi_j8', 'perms': git.GitPerm.TREE, # we only add children's sha1_git here, in reality, # it's a full dict of hashes. - 'children': __children_hashes(b'/tmp/tmp7w3oi_j8') + 'children': _children_hashes(b'/tmp/tmp7w3oi_j8') }, { 'type': git.GitType.TREE, 'name': b'sample-folder', 'sha1_git': b'\xe8\xb0\xf1Fj\xf8`\x8c\x8a?\xb9\x87\x9d\xb1r\xb8\x87\xe8\x07Y', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder', 'perms': git.GitPerm.TREE, - 'children': __children_hashes( + 'children': _children_hashes( b'/tmp/tmp7w3oi_j8/sample-folder') }, { 'type': git.GitType.TREE, 'name': b'empty-folder', 'sha1_git': b'K\x82]\xc6B\xcbn\xb9\xa0`\xe5K\xf8\xd6\x92\x88\xfb\xeeI\x04', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/empty-folder', 'perms': git.GitPerm.TREE, - 'children': __children_hashes( + 'children': _children_hashes( b'/tmp/tmp7w3oi_j8/sample-folder/empty-folder') }, { 'type': git.GitType.TREE, 'name': b'bar', 'sha1_git': b'<\x1fW\x83\x94\xf4b?t\xa0\xba\x7f\xe7ar\x9fY\xfcn\xc4', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/bar', 'perms': git.GitPerm.TREE, - 'children': __children_hashes( + 'children': _children_hashes( b'/tmp/tmp7w3oi_j8/sample-folder/bar') }, { 'type': git.GitType.TREE, 'name': b'barfoo', 'sha1_git': b'\xc3\x02\x0fk\xf15\xa3\x8cm\xf3\xaf\xeb_\xb3\x822\xc5\xe0p\x87', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/bar/barfoo', 'perms': git.GitPerm.TREE, - 'children': __children_hashes( + 'children': _children_hashes( b'/tmp/tmp7w3oi_j8/sample-folder/bar/barfoo'), }, { 'type': git.GitType.TREE, 'name': b'foo', 'sha1_git': b'+A\xc4\x0f\r\x1f\xbf\xfc\xba\x12I}\xb7\x1f\xba\x83\xfc\xca\x96\xe5', # noqa 'path': b'/tmp/tmp7w3oi_j8/sample-folder/foo', 'perms': git.GitPerm.TREE, - 'children': __children_hashes( + 'children': _children_hashes( b'/tmp/tmp7w3oi_j8/sample-folder/foo') }, ] expected_sha1_trees = list( ((c['sha1_git'], git.GitType.TREE, c['children']) for c in expected_trees)) # when actual_sha1_trees = list( - ((c['sha1_git'], c['type'], __children_hashes(c['path'])) + ((c['sha1_git'], c['type'], _children_hashes(c['path'])) for c in git.objects_per_type(git.GitType.TREE, self.objects))) self.assertEquals(len(actual_sha1_trees), len(expected_sha1_trees)) for e in actual_sha1_trees: self.assertTrue(e in expected_sha1_trees) class TestComputeHashesFromDirectory(WithSampleFolderChecksums, GitHashWalkArborescenceTree, unittest.TestCase): def __adapt_object_to_rootpath(self, rootpath): def _replace_slash(s, rootpath=self.rootkey, newrootpath=rootpath): return s.replace(rootpath, newrootpath) def _update_children(children): return set((_replace_slash(c) for c in children)) # given expected_objects = {} for path, v in self.objects.items(): p = _replace_slash(path) v['checksums']['path'] = _replace_slash(v['checksums']['path']) v['checksums']['name'] = os.path.basename(v['checksums']['path']) if 'children' in v: v['children'] = _update_children(v['children']) expected_objects[p] = v return expected_objects @istest def compute_hashes_from_directory_default(self): # given expected_objects = self.__adapt_object_to_rootpath(self.tmp_root_path) # when actual_hashes = git.compute_hashes_from_directory(self.tmp_root_path) # then self.assertEquals(actual_hashes, expected_objects) @istest def compute_hashes_from_directory_no_empty_folder(self): # given def _replace_slash(s, rootpath=self.rootkey, newrootpath=self.tmp_root_path): return s.replace(rootpath, newrootpath) expected_objects = self.__adapt_object_to_rootpath(self.tmp_root_path) # when actual_hashes = git.compute_hashes_from_directory( self.tmp_root_path, remove_empty_folder=True) # then # One folder less, so plenty of hashes are different now self.assertNotEquals(actual_hashes, expected_objects) keys = set(actual_hashes.keys()) assert (b'/tmp/tmp7w3oi_j8/sample-folder/empty-folder' in self.objects.keys()) new_empty_folder_path = _replace_slash( b'/tmp/tmp7w3oi_j8/sample-folder/empty-folder') self.assertNotIn(new_empty_folder_path, keys) self.assertEqual(len(keys), len(expected_objects.keys()) - 1) @istest def compute_hashes_from_directory_ignore_some_folder(self): # given def _replace_slash(s, rootpath=self.rootkey, newrootpath=self.tmp_root_path): return s.replace(rootpath, newrootpath) ignore_path = b'/tmp/tmp7w3oi_j8/sample-folder' # when actual_hashes = git.compute_hashes_from_directory( self.tmp_root_path, dir_ok_fn=lambda dirpath: b'sample-folder' not in dirpath) # then # One entry less, so plenty of hashes are different now keys = set(actual_hashes.keys()) assert ignore_path in self.objects.keys() new_ignore_path = _replace_slash(ignore_path) self.assertNotIn(new_ignore_path, keys) # top level directory contains the folder to ignore self.assertEqual(len(keys), 1) diff --git a/swh/model/tests/test_identifiers.py b/swh/model/tests/test_identifiers.py index e1adfea..16a34bb 100644 --- a/swh/model/tests/test_identifiers.py +++ b/swh/model/tests/test_identifiers.py @@ -1,641 +1,651 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import binascii import datetime import unittest from nose.tools import istest from swh.model import hashutil, identifiers class UtilityFunctionsIdentifier(unittest.TestCase): def setUp(self): self.str_id = 'c2e41aae41ac17bd4a650770d6ee77f62e52235b' self.bytes_id = binascii.unhexlify(self.str_id) self.bad_type_id = object() @istest def identifier_to_bytes(self): for id in [self.str_id, self.bytes_id]: self.assertEqual(identifiers.identifier_to_bytes(id), self.bytes_id) # wrong length with self.assertRaises(ValueError) as cm: identifiers.identifier_to_bytes(id[:-2]) self.assertIn('length', str(cm.exception)) with self.assertRaises(ValueError) as cm: identifiers.identifier_to_bytes(self.bad_type_id) self.assertIn('type', str(cm.exception)) @istest def identifier_to_str(self): for id in [self.str_id, self.bytes_id]: self.assertEqual(identifiers.identifier_to_str(id), self.str_id) # wrong length with self.assertRaises(ValueError) as cm: identifiers.identifier_to_str(id[:-2]) self.assertIn('length', str(cm.exception)) with self.assertRaises(ValueError) as cm: identifiers.identifier_to_str(self.bad_type_id) self.assertIn('type', str(cm.exception)) class UtilityFunctionsDateOffset(unittest.TestCase): def setUp(self): - self.date = datetime.datetime( - 2015, 11, 22, 16, 33, 56, tzinfo=datetime.timezone.utc) - self.date_int = int(self.date.timestamp()) - self.date_repr = b'1448210036' - - self.date_microseconds = datetime.datetime( - 2015, 11, 22, 16, 33, 56, 2342, tzinfo=datetime.timezone.utc) - self.date_microseconds_float = self.date_microseconds.timestamp() - self.date_microseconds_repr = b'1448210036.002342' + self.dates = { + b'1448210036': { + 'seconds': 1448210036, + 'microseconds': 0, + }, + b'1448210036.002342': { + 'seconds': 1448210036, + 'microseconds': 2342, + }, + b'1448210036.12': { + 'seconds': 1448210036, + 'microseconds': 120000, + } + } + self.broken_dates = [ + 1448210036.12, + ] self.offsets = { 0: b'+0000', -630: b'-1030', 800: b'+1320', } @istest def format_date(self): - for date in [self.date, self.date_int]: - self.assertEqual(identifiers.format_date(date), self.date_repr) + for date_repr, date in self.dates.items(): + self.assertEqual(identifiers.format_date(date), date_repr) - for date in [self.date_microseconds, self.date_microseconds_float]: - self.assertEqual(identifiers.format_date(date), - self.date_microseconds_repr) + @istest + def format_date_fail(self): + for date in self.broken_dates: + with self.assertRaises(ValueError): + identifiers.format_date(date) @istest def format_offset(self): for offset, res in self.offsets.items(): self.assertEqual(identifiers.format_offset(offset), res) class ContentIdentifier(unittest.TestCase): def setUp(self): self.content = { 'status': 'visible', 'length': 5, 'data': b'1984\n', 'ctime': datetime.datetime(2015, 11, 22, 16, 33, 56, tzinfo=datetime.timezone.utc), } self.content_id = hashutil.hash_data(self.content['data']) @istest def content_identifier(self): self.assertEqual(identifiers.content_identifier(self.content), self.content_id) class DirectoryIdentifier(unittest.TestCase): def setUp(self): self.directory = { 'id': 'c2e41aae41ac17bd4a650770d6ee77f62e52235b', 'entries': [ { 'type': 'file', 'perms': 33188, 'name': b'README', 'target': '37ec8ea2110c0b7a32fbb0e872f6e7debbf95e21' }, { 'type': 'file', 'perms': 33188, 'name': b'Rakefile', 'target': '3bb0e8592a41ae3185ee32266c860714980dbed7' }, { 'type': 'dir', 'perms': 16384, 'name': b'app', 'target': '61e6e867f5d7ba3b40540869bc050b0c4fed9e95' }, { 'type': 'file', 'perms': 33188, 'name': b'1.megabyte', 'target': '7c2b2fbdd57d6765cdc9d84c2d7d333f11be7fb3' }, { 'type': 'dir', 'perms': 16384, 'name': b'config', 'target': '591dfe784a2e9ccc63aaba1cb68a765734310d98' }, { 'type': 'dir', 'perms': 16384, 'name': b'public', 'target': '9588bf4522c2b4648bfd1c61d175d1f88c1ad4a5' }, { 'type': 'file', 'perms': 33188, 'name': b'development.sqlite3', 'target': 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391' }, { 'type': 'dir', 'perms': 16384, 'name': b'doc', 'target': '154705c6aa1c8ead8c99c7915373e3c44012057f' }, { 'type': 'dir', 'perms': 16384, 'name': b'db', 'target': '85f157bdc39356b7bc7de9d0099b4ced8b3b382c' }, { 'type': 'dir', 'perms': 16384, 'name': b'log', 'target': '5e3d3941c51cce73352dff89c805a304ba96fffe' }, { 'type': 'dir', 'perms': 16384, 'name': b'script', 'target': '1b278423caf176da3f3533592012502aa10f566c' }, { 'type': 'dir', 'perms': 16384, 'name': b'test', 'target': '035f0437c080bfd8711670b3e8677e686c69c763' }, { 'type': 'dir', 'perms': 16384, 'name': b'vendor', 'target': '7c0dc9ad978c1af3f9a4ce061e50f5918bd27138' }, { 'type': 'rev', 'perms': 57344, 'name': b'will_paginate', 'target': '3d531e169db92a16a9a8974f0ae6edf52e52659e' } ], } self.empty_directory = { 'id': '4b825dc642cb6eb9a060e54bf8d69288fbee4904', 'entries': [], } @istest def dir_identifier(self): self.assertEqual( identifiers.directory_identifier(self.directory), self.directory['id']) @istest def dir_identifier_empty_directory(self): self.assertEqual( identifiers.directory_identifier(self.empty_directory), self.empty_directory['id']) class RevisionIdentifier(unittest.TestCase): def setUp(self): linus_tz = datetime.timezone(datetime.timedelta(minutes=-420)) gpgsig = b'''\ -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.13 (Darwin) iQIcBAABAgAGBQJVJcYsAAoJEBiY3kIkQRNJVAUQAJ8/XQIfMqqC5oYeEFfHOPYZ L7qy46bXHVBa9Qd8zAJ2Dou3IbI2ZoF6/Et89K/UggOycMlt5FKV/9toWyuZv4Po L682wonoxX99qvVTHo6+wtnmYO7+G0f82h+qHMErxjP+I6gzRNBvRr+SfY7VlGdK wikMKOMWC5smrScSHITnOq1Ews5pe3N7qDYMzK0XVZmgDoaem4RSWMJs4My/qVLN e0CqYWq2A22GX7sXl6pjneJYQvcAXUX+CAzp24QnPSb+Q22Guj91TcxLFcHCTDdn qgqMsEyMiisoglwrCbO+D+1xq9mjN9tNFWP66SQ48mrrHYTBV5sz9eJyDfroJaLP CWgbDTgq6GzRMehHT3hXfYS5NNatjnhkNISXR7pnVP/obIi/vpWh5ll6Gd8q26z+ a/O41UzOaLTeNI365MWT4/cnXohVLRG7iVJbAbCxoQmEgsYMRc/pBAzWJtLfcB2G jdTswYL6+MUdL8sB9pZ82D+BP/YAdHe69CyTu1lk9RT2pYtI/kkfjHubXBCYEJSG +VGllBbYG6idQJpyrOYNRJyrDi9yvDJ2W+S0iQrlZrxzGBVGTB/y65S8C+2WTBcE lf1Qb5GDsQrZWgD+jtWTywOYHtCBwyCKSAXxSARMbNPeak9WPlcW/Jmu+fUcMe2x dg1KdHOa34shrKDaOVzW =od6m -----END PGP SIGNATURE-----''' self.revision = { 'id': 'bc0195aad0daa2ad5b0d76cce22b167bc3435590', 'directory': '85a74718d377195e1efd0843ba4f3260bad4fe07', 'parents': ['01e2d0627a9a6edb24c37db45db5ecb31e9de808'], 'author': { 'name': b'Linus Torvalds', 'email': b'torvalds@linux-foundation.org', }, 'date': datetime.datetime(2015, 7, 12, 15, 10, 30, tzinfo=linus_tz), 'committer': { 'name': b'Linus Torvalds', 'email': b'torvalds@linux-foundation.org', }, 'committer_date': datetime.datetime(2015, 7, 12, 15, 10, 30, tzinfo=linus_tz), 'message': b'Linux 4.2-rc2\n', } self.revision_none_metadata = { 'id': 'bc0195aad0daa2ad5b0d76cce22b167bc3435590', 'directory': '85a74718d377195e1efd0843ba4f3260bad4fe07', 'parents': ['01e2d0627a9a6edb24c37db45db5ecb31e9de808'], 'author': { 'name': b'Linus Torvalds', 'email': b'torvalds@linux-foundation.org', }, 'date': datetime.datetime(2015, 7, 12, 15, 10, 30, tzinfo=linus_tz), 'committer': { 'name': b'Linus Torvalds', 'email': b'torvalds@linux-foundation.org', }, 'committer_date': datetime.datetime(2015, 7, 12, 15, 10, 30, tzinfo=linus_tz), 'message': b'Linux 4.2-rc2\n', 'metadata': None, } self.synthetic_revision = { 'id': b'\xb2\xa7\xe1&\x04\x92\xe3D\xfa\xb3\xcb\xf9\x1b\xc1<\x91' b'\xe0T&\xfd', 'author': { 'name': b'Software Heritage', 'email': b'robot@softwareheritage.org', }, 'date': { - 'timestamp': 1437047495.0, + 'timestamp': {'seconds': 1437047495}, 'offset': 0, 'negative_utc': False, }, 'type': 'tar', 'committer': { 'name': b'Software Heritage', 'email': b'robot@softwareheritage.org', }, 'committer_date': 1437047495, 'synthetic': True, 'parents': [None], 'message': b'synthetic revision message\n', 'directory': b'\xd1\x1f\x00\xa6\xa0\xfe\xa6\x05SA\xd2U\x84\xb5\xa9' b'e\x16\xc0\xd2\xb8', 'metadata': {'original_artifact': [ {'archive_type': 'tar', 'name': 'gcc-5.2.0.tar.bz2', 'sha1_git': '39d281aff934d44b439730057e55b055e206a586', 'sha1': 'fe3f5390949d47054b613edc36c557eb1d51c18e', 'sha256': '5f835b04b5f7dd4f4d2dc96190ec1621b8d89f' '2dc6f638f9f8bc1b1014ba8cad'}]}, } # cat commit.txt | git hash-object -t commit --stdin self.revision_with_extra_headers = { 'id': '010d34f384fa99d047cdd5e2f41e56e5c2feee45', 'directory': '85a74718d377195e1efd0843ba4f3260bad4fe07', 'parents': ['01e2d0627a9a6edb24c37db45db5ecb31e9de808'], 'author': { 'name': b'Linus Torvalds', 'email': b'torvalds@linux-foundation.org', 'fullname': b'Linus Torvalds ', }, 'date': datetime.datetime(2015, 7, 12, 15, 10, 30, tzinfo=linus_tz), 'committer': { 'name': b'Linus Torvalds', 'email': b'torvalds@linux-foundation.org', 'fullname': b'Linus Torvalds ', }, 'committer_date': datetime.datetime(2015, 7, 12, 15, 10, 30, tzinfo=linus_tz), 'message': b'Linux 4.2-rc2\n', 'metadata': { 'extra_headers': [ ['svn-repo-uuid', '046f1af7-66c2-d61b-5410-ce57b7db7bff'], ['svn-revision', 10], ] } } self.revision_with_gpgsig = { 'id': '44cc742a8ca17b9c279be4cc195a93a6ef7a320e', 'directory': 'b134f9b7dc434f593c0bab696345548b37de0558', 'parents': ['689664ae944b4692724f13b709a4e4de28b54e57', 'c888305e1efbaa252d01b4e5e6b778f865a97514'], 'author': { 'name': b'Jiang Xin', 'email': b'worldhello.net@gmail.com', 'fullname': b'Jiang Xin ', }, 'date': { - 'timestamp': '1428538899', + 'timestamp': 1428538899, 'offset': 480, }, 'committer': { 'name': b'Jiang Xin', 'email': b'worldhello.net@gmail.com', }, 'committer_date': { - 'timestamp': '1428538899', + 'timestamp': 1428538899, 'offset': 480, }, 'metadata': { 'extra_headers': [ ['gpgsig', gpgsig], ], }, 'message': b'''Merge branch 'master' of git://github.com/alexhenrie/git-po * 'master' of git://github.com/alexhenrie/git-po: l10n: ca.po: update translation ''' } self.revision_no_message = { 'id': '4cfc623c9238fa92c832beed000ce2d003fd8333', 'directory': 'b134f9b7dc434f593c0bab696345548b37de0558', 'parents': ['689664ae944b4692724f13b709a4e4de28b54e57', 'c888305e1efbaa252d01b4e5e6b778f865a97514'], 'author': { 'name': b'Jiang Xin', 'email': b'worldhello.net@gmail.com', 'fullname': b'Jiang Xin ', }, 'date': { - 'timestamp': '1428538899', + 'timestamp': 1428538899, 'offset': 480, }, 'committer': { 'name': b'Jiang Xin', 'email': b'worldhello.net@gmail.com', }, 'committer_date': { - 'timestamp': '1428538899', + 'timestamp': 1428538899, 'offset': 480, }, 'message': None, } self.revision_empty_message = { 'id': '7442cd78bd3b4966921d6a7f7447417b7acb15eb', 'directory': 'b134f9b7dc434f593c0bab696345548b37de0558', 'parents': ['689664ae944b4692724f13b709a4e4de28b54e57', 'c888305e1efbaa252d01b4e5e6b778f865a97514'], 'author': { 'name': b'Jiang Xin', 'email': b'worldhello.net@gmail.com', 'fullname': b'Jiang Xin ', }, 'date': { - 'timestamp': '1428538899', + 'timestamp': 1428538899, 'offset': 480, }, 'committer': { 'name': b'Jiang Xin', 'email': b'worldhello.net@gmail.com', }, 'committer_date': { - 'timestamp': '1428538899', + 'timestamp': 1428538899, 'offset': 480, }, 'message': b'', } self.revision_only_fullname = { 'id': '010d34f384fa99d047cdd5e2f41e56e5c2feee45', 'directory': '85a74718d377195e1efd0843ba4f3260bad4fe07', 'parents': ['01e2d0627a9a6edb24c37db45db5ecb31e9de808'], 'author': { 'fullname': b'Linus Torvalds ', }, 'date': datetime.datetime(2015, 7, 12, 15, 10, 30, tzinfo=linus_tz), 'committer': { 'fullname': b'Linus Torvalds ', }, 'committer_date': datetime.datetime(2015, 7, 12, 15, 10, 30, tzinfo=linus_tz), 'message': b'Linux 4.2-rc2\n', 'metadata': { 'extra_headers': [ ['svn-repo-uuid', '046f1af7-66c2-d61b-5410-ce57b7db7bff'], ['svn-revision', 10], ] } } @istest def revision_identifier(self): self.assertEqual( identifiers.revision_identifier(self.revision), identifiers.identifier_to_str(self.revision['id']), ) @istest def revision_identifier_none_metadata(self): self.assertEqual( identifiers.revision_identifier(self.revision_none_metadata), identifiers.identifier_to_str(self.revision_none_metadata['id']), ) @istest def revision_identifier_synthetic(self): self.assertEqual( identifiers.revision_identifier(self.synthetic_revision), identifiers.identifier_to_str(self.synthetic_revision['id']), ) @istest def revision_identifier_with_extra_headers(self): self.assertEqual( identifiers.revision_identifier( self.revision_with_extra_headers), identifiers.identifier_to_str( self.revision_with_extra_headers['id']), ) @istest def revision_identifier_with_gpgsig(self): self.assertEqual( identifiers.revision_identifier( self.revision_with_gpgsig), identifiers.identifier_to_str( self.revision_with_gpgsig['id']), ) @istest def revision_identifier_no_message(self): self.assertEqual( identifiers.revision_identifier( self.revision_no_message), identifiers.identifier_to_str( self.revision_no_message['id']), ) @istest def revision_identifier_empty_message(self): self.assertEqual( identifiers.revision_identifier( self.revision_empty_message), identifiers.identifier_to_str( self.revision_empty_message['id']), ) @istest def revision_identifier_only_fullname(self): self.assertEqual( identifiers.revision_identifier( self.revision_only_fullname), identifiers.identifier_to_str( self.revision_only_fullname['id']), ) class ReleaseIdentifier(unittest.TestCase): def setUp(self): linus_tz = datetime.timezone(datetime.timedelta(minutes=-420)) self.release = { 'id': '2b10839e32c4c476e9d94492756bb1a3e1ec4aa8', 'target': b't\x1b"R\xa5\xe1Ml`\xa9\x13\xc7z`\x99\xab\xe7:\x85J', 'target_type': 'revision', 'name': b'v2.6.14', 'author': { 'name': b'Linus Torvalds', 'email': b'torvalds@g5.osdl.org', }, 'date': datetime.datetime(2005, 10, 27, 17, 2, 33, tzinfo=linus_tz), 'message': b'''\ Linux 2.6.14 release -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.1 (GNU/Linux) iD8DBQBDYWq6F3YsRnbiHLsRAmaeAJ9RCez0y8rOBbhSv344h86l/VVcugCeIhO1 wdLOnvj91G4wxYqrvThthbE= =7VeT -----END PGP SIGNATURE----- ''', 'synthetic': False, } self.release_no_author = { 'id': b'&y\x1a\x8b\xcf\x0em3\xf4:\xefv\x82\xbd\xb5U#mV\xde', 'target': '9ee1c939d1cb936b1f98e8d81aeffab57bae46ab', 'target_type': 'revision', 'name': b'v2.6.12', 'message': b'''\ This is the final 2.6.12 release -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.2.4 (GNU/Linux) iD8DBQBCsykyF3YsRnbiHLsRAvPNAJ482tCZwuxp/bJRz7Q98MHlN83TpACdHr37 o6X/3T+vm8K3bf3driRr34c= =sBHn -----END PGP SIGNATURE----- ''', 'synthetic': False, } self.release_no_message = { 'id': 'b6f4f446715f7d9543ef54e41b62982f0db40045', 'target': '9ee1c939d1cb936b1f98e8d81aeffab57bae46ab', 'target_type': 'revision', 'name': b'v2.6.12', 'author': { 'name': b'Linus Torvalds', 'email': b'torvalds@g5.osdl.org', }, 'date': datetime.datetime(2005, 10, 27, 17, 2, 33, tzinfo=linus_tz), 'message': None, } self.release_empty_message = { 'id': '71a0aea72444d396575dc25ac37fec87ee3c6492', 'target': '9ee1c939d1cb936b1f98e8d81aeffab57bae46ab', 'target_type': 'revision', 'name': b'v2.6.12', 'author': { 'name': b'Linus Torvalds', 'email': b'torvalds@g5.osdl.org', }, 'date': datetime.datetime(2005, 10, 27, 17, 2, 33, tzinfo=linus_tz), 'message': b'', } self.release_negative_utc = { 'id': '97c8d2573a001f88e72d75f596cf86b12b82fd01', 'name': b'20081029', 'target': '54e9abca4c77421e2921f5f156c9fe4a9f7441c7', 'target_type': 'revision', 'date': { - 'timestamp': 1225281976.0, + 'timestamp': {'seconds': 1225281976}, 'offset': 0, 'negative_utc': True, }, 'author': { 'name': b'Otavio Salvador', 'email': b'otavio@debian.org', 'id': 17640, }, 'synthetic': False, 'message': b'tagging version 20081029\n\nr56558\n', } @istest def release_identifier(self): self.assertEqual( identifiers.release_identifier(self.release), identifiers.identifier_to_str(self.release['id']) ) @istest def release_identifier_no_author(self): self.assertEqual( identifiers.release_identifier(self.release_no_author), identifiers.identifier_to_str(self.release_no_author['id']) ) @istest def release_identifier_no_message(self): self.assertEqual( identifiers.release_identifier(self.release_no_message), identifiers.identifier_to_str(self.release_no_message['id']) ) @istest def release_identifier_empty_message(self): self.assertEqual( identifiers.release_identifier(self.release_empty_message), identifiers.identifier_to_str(self.release_empty_message['id']) ) @istest def release_identifier_negative_utc(self): self.assertEqual( identifiers.release_identifier(self.release_negative_utc), identifiers.identifier_to_str(self.release_negative_utc['id']) ) diff --git a/version.txt b/version.txt index df588ce..ec62758 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.12-0-gcec445d \ No newline at end of file +v0.0.13-0-g58c5a24 \ No newline at end of file