diff --git a/PKG-INFO b/PKG-INFO index 02c5a5d..f3afd9e 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.loader.dir -Version: 0.0.17 +Version: 0.0.18 Summary: Software Heritage Directory Loader Home-page: https://forge.softwareheritage.org/diffusion/DLDDIR Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/README-dev.md b/README-dev.md deleted file mode 100644 index b52e61a..0000000 --- a/README-dev.md +++ /dev/null @@ -1,59 +0,0 @@ -Git sha1 computation --------------------- - -Document to describe how the git sha1 computation takes place. - -### commit/revision - -sha1 git commit/revision computation: - - commit `size`\0 - tree `sha1-git-tree-and-subtree-in-plain-hex-string` - (parent `commit-parent`) - author `name` <`email`> `date-ts` `date-offset` - committer `name` <`email`> `date-ts` `date-offset` - - `commit-message` - - -Notes: -- () denotes optional entry. Indeed, first commit does not contain any parent commit. -- empty line at the end of the commit message -- timestamp example: 1444054085 -- date offset for example: +0200 - -### directory/tree - -sha1 git directory/tree computation: - - tree `tree-size`\0 - \0... \0... - - -Notes: -- no newline separator between tree entries -- no empty newline at the end of the tree entries -- tree content header size is the length of the content -- The tree entries are ordered according to bytes in their properties. - -Note: Tree entries referencing trees are sorted as if their name have a trailing / -at their end. - -Possible permissions are: -- 100644 - file -- 40000 - directory -- 100755 - executable file -- 120000 - symbolink link -- 160000 - git link (relative to submodule) - -### content/file - -sha1 git content computation: - - blob `blob-size`\0 - `blob-content` - -Notes: -- no newline at the end of the blob content - -Compress with DEFLATE and compute sha1 diff --git a/debian/control b/debian/control index 651bd31..5d68b13 100644 --- a/debian/control +++ b/debian/control @@ -1,25 +1,23 @@ Source: swh-loader-dir Maintainer: Software Heritage developers Section: python Priority: optional Build-Depends: debhelper (>= 9), dh-python, python3-all, python3-nose, python3-setuptools, - python3-swh.core (>= 0.0.14), - python3-swh.model (>= 0.0.4), + python3-swh.core (>= 0.0.14~), + python3-swh.model (>= 0.0.4~), python3-swh.scheduler, - python3-swh.storage (>= 0.0.31), + python3-swh.storage (>= 0.0.31~), + python3-swh.loader.core (>= 0.0.5~), python3-vcversioner Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DLDDIR/ Package: python3-swh.loader.dir Architecture: all -Depends: python3-swh.core (>= 0.0.14), - python3-swh.model (>= 0.0.4), - python3-swh.storage (>= 0.0.31), - ${misc:Depends}, +Depends: ${misc:Depends}, ${python3:Depends} Description: Software Heritage Directory Loader diff --git a/debian/copyright b/debian/copyright index 81d037d..9430897 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,22 +1,22 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Files: * -Copyright: 2015 The Software Heritage developers +Copyright: 2015-2016 The Software Heritage developers License: GPL-3+ License: GPL-3+ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. . This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. . You should have received a copy of the GNU General Public License along with this program. If not, see . . On Debian systems, the complete text of the GNU General Public License version 3 can be found in `/usr/share/common-licenses/GPL-3'. diff --git a/requirements.txt b/requirements.txt index e2b17fa..1feb592 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ # Add here external Python modules dependencies, one per line. Module names # should match https://pypi.python.org/pypi names. For the full spec or # dependency lines, see https://pip.readthedocs.org/en/1.1/requirements.html vcversioner swh.core >= 0.0.14 swh.model >= 0.0.4 swh.scheduler swh.storage >= 0.0.31 +swh.loader.core >= 0.0.5 retrying diff --git a/resources/dir.ini b/resources/dir.ini new file mode 100644 index 0000000..3d41520 --- /dev/null +++ b/resources/dir.ini @@ -0,0 +1,15 @@ +[main] +storage_class = remote_storage +storage_args = http://localhost:5000/ +send_contents = True +send_directories = True +send_revisions = True +send_releases = True +send_occurrences = True +content_packet_size = 10000 +content_packet_size_bytes = 1073741824 +content_packet_block_size_bytes = 104857600 +directory_packet_size = 25000 +revision_packet_size = 100000 +release_packet_size = 100000 +occurrence_packet_size = 100000 diff --git a/swh.loader.dir.egg-info/PKG-INFO b/swh.loader.dir.egg-info/PKG-INFO index 02c5a5d..f3afd9e 100644 --- a/swh.loader.dir.egg-info/PKG-INFO +++ b/swh.loader.dir.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.loader.dir -Version: 0.0.17 +Version: 0.0.18 Summary: Software Heritage Directory Loader Home-page: https://forge.softwareheritage.org/diffusion/DLDDIR Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh.loader.dir.egg-info/SOURCES.txt b/swh.loader.dir.egg-info/SOURCES.txt index 2049218..2b9fb74 100644 --- a/swh.loader.dir.egg-info/SOURCES.txt +++ b/swh.loader.dir.egg-info/SOURCES.txt @@ -1,33 +1,30 @@ .gitignore AUTHORS LICENSE MANIFEST.in Makefile Makefile.local README -README-dev.md requirements.txt setup.py version.txt bin/swh-check-missing-objects.py bin/swh-loader-dir debian/changelog debian/compat debian/control debian/copyright debian/rules debian/source/format +resources/dir.ini resources/loader/dir.ini scratch/walking.py swh.loader.dir.egg-info/PKG-INFO swh.loader.dir.egg-info/SOURCES.txt swh.loader.dir.egg-info/dependency_links.txt swh.loader.dir.egg-info/requires.txt swh.loader.dir.egg-info/top_level.txt swh/loader/dir/__init__.py -swh/loader/dir/converters.py -swh/loader/dir/git.py swh/loader/dir/loader.py swh/loader/dir/tasks.py -swh/loader/dir/tests/test_converters.py swh/loader/dir/tests/test_loader.py \ No newline at end of file diff --git a/swh.loader.dir.egg-info/requires.txt b/swh.loader.dir.egg-info/requires.txt index ec5b428..4e31f97 100644 --- a/swh.loader.dir.egg-info/requires.txt +++ b/swh.loader.dir.egg-info/requires.txt @@ -1,6 +1,7 @@ retrying swh.core>=0.0.14 +swh.loader.core>=0.0.5 swh.model>=0.0.4 swh.scheduler swh.storage>=0.0.31 vcversioner diff --git a/swh/loader/dir/converters.py b/swh/loader/dir/converters.py deleted file mode 100644 index 33572f2..0000000 --- a/swh/loader/dir/converters.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright (C) 2015 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -"""Convert dir objects to dictionaries suitable for swh.storage""" - -import datetime -import os - -from swh.model.hashutil import hash_to_hex - -from swh.model import git - - -def to_datetime(ts): - """Convert a timestamp to utc datetime. - - """ - return datetime.datetime.utcfromtimestamp(ts).replace( - tzinfo=datetime.timezone.utc) - - -def format_to_minutes(offset_str): - """Convert a git string timezone format string (e.g +0200, -0310) to minutes. - - Args: - offset_str: a string representing an offset. - - Returns: - A positive or negative number of minutes of such input - - """ - sign = offset_str[0] - hours = int(offset_str[1:3]) - minutes = int(offset_str[3:]) + (hours * 60) - return minutes if sign == '+' else -1 * minutes - - -def blob_to_content(obj, log=None, max_content_size=None, - origin_id=None): - """Convert obj to a swh storage content. - - Note: - - If obj represents a link, the length and data are already - provided so we use them directly. - - 'data' is returned only if max_content_size is not reached. - - Returns: - obj converted to content as a dictionary. - - """ - filepath = obj['path'] - if 'length' in obj: # link already has it - size = obj['length'] - else: - size = os.lstat(filepath).st_size - - ret = { - 'sha1': obj['sha1'], - 'sha256': obj['sha256'], - 'sha1_git': obj['sha1_git'], - 'length': size, - 'perms': obj['perms'].value, - 'type': obj['type'].value, - } - - if max_content_size and size > max_content_size: - if log: - log.info('Skipping content %s, too large (%s > %s)' % - (hash_to_hex(obj['sha1_git']), - size, - max_content_size)) - ret.update({'status': 'absent', - 'reason': 'Content too large', - 'origin': origin_id}) - return ret - - if 'data' in obj: # link already has it - data = obj['data'] - else: - data = open(filepath, 'rb').read() - - ret.update({ - 'data': data, - 'status': 'visible' - }) - - return ret - - -# Map of type to swh types -_entry_type_map = { - git.GitType.TREE: 'dir', - git.GitType.BLOB: 'file', - git.GitType.COMM: 'rev', -} - - -def tree_to_directory(tree, objects, log=None): - """Format a tree as a directory - - """ - entries = [] - for entry in objects[tree['path']]: - entries.append({ - 'type': _entry_type_map[entry['type']], - 'perms': int(entry['perms'].value), - 'name': entry['name'], - 'target': entry['sha1_git'] - }) - - return { - 'id': tree['sha1_git'], - 'entries': entries - } - - -def commit_to_revision(commit, objects, log=None): - """Format a commit as a revision. - - """ - upper_directory = objects[git.ROOT_TREE_KEY][0] - return { - 'date': { - 'timestamp': commit['author_date'], - 'offset': format_to_minutes(commit['author_offset']), - }, - 'committer_date': { - 'timestamp': commit['committer_date'], - 'offset': format_to_minutes(commit['committer_offset']), - }, - 'type': commit['type'], - 'directory': upper_directory['sha1_git'], - 'message': commit['message'].encode('utf-8'), - 'author': { - 'name': commit['author_name'].encode('utf-8'), - 'email': commit['author_email'].encode('utf-8'), - }, - 'committer': { - 'name': commit['committer_name'].encode('utf-8'), - 'email': commit['committer_email'].encode('utf-8'), - }, - 'synthetic': True, - 'metadata': commit['metadata'], - 'parents': [], - } - - -def annotated_tag_to_release(release, log=None): - """Format a swh release. - - """ - return { - 'target': release['target'], - 'target_type': release['target_type'], - 'name': release['name'].encode('utf-8'), - 'message': release['comment'].encode('utf-8'), - 'date': { - 'timestamp': release['date'], - 'offset': format_to_minutes(release['offset']), - }, - 'author': { - 'name': release['author_name'].encode('utf-8'), - 'email': release['author_email'].encode('utf-8'), - }, - 'synthetic': True, - } - - -def ref_to_occurrence(ref): - """Format a reference as an occurrence""" - occ = ref.copy() - if 'branch' in ref: - branch = ref['branch'] - if isinstance(branch, str): - occ['branch'] = branch.encode('utf-8') - else: - occ['branch'] = branch - return occ diff --git a/swh/loader/dir/git.py b/swh/loader/dir/git.py deleted file mode 100644 index 155ba98..0000000 --- a/swh/loader/dir/git.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright (C) 2015 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - - -import os - -from enum import Enum - -from swh.model import hashutil, identifiers - - -ROOT_TREE_KEY = b'' - - -class GitType(Enum): - BLOB = b'blob' - TREE = b'tree' - EXEC = b'exec' - LINK = b'link' - COMM = b'commit' - RELE = b'release' - REFS = b'ref' - - -class GitPerm(Enum): - BLOB = b'100644' - TREE = b'40000' - EXEC = b'100755' - LINK = b'120000' - - -def compute_directory_git_sha1(dirpath, hashes): - """Compute a directory git sha1 for a dirpath. - - Args: - dirpath: the directory's absolute path - hashes: list of tree entries with keys: - - sha1_git: the tree entry's sha1 - - name: file or subdir's name - - perms: the tree entry's sha1 permissions - - Returns: - the binary sha1 of the dictionary's identifier - - Assumes: - Every path exists in hashes. - - """ - directory = { - 'entries': - [ - { - 'name': entry['name'], - 'perms': int(entry['perms'].value, 8), - 'target': entry['sha1_git'], - 'type': 'dir' if entry['perms'] == GitPerm.TREE else 'file', - } - for entry in hashes[dirpath] - ] - } - return hashutil.hash_to_bytes(identifiers.directory_identifier(directory)) - - -def compute_revision_sha1_git(revision): - """Compute a revision sha1 git from its dict representation. - - Args: - revision: Additional dictionary information needed to compute a - synthetic - revision. Following keys are expected: - - author - - date - - committer - - committer_date - - message - - type - - directory: binary form of the tree hash - - Returns: - revision sha1 in bytes - - # FIXME: beware, bytes output from storage api - - """ - return hashutil.hash_to_bytes(identifiers.revision_identifier(revision)) - - -def compute_release_sha1_git(release): - """Compute a release sha1 git from its dict representation. - - Args: - release: Additional dictionary information needed to compute a - synthetic release. Following keys are expected: - - name - - message - - date - - author - - revision: binary form of the sha1_git revision targeted by this - - Returns: - release sha1 in bytes - - """ - return hashutil.hash_to_bytes(identifiers.release_identifier(release)) - - -def compute_link_metadata(linkpath): - """Given a linkpath, compute the git metadata. - - Args: - linkpath: absolute pathname of the link - - Returns: - Dictionary of values: - - name: basename of the link - - perms: git permission for link - - type: git type for link - """ - data = os.readlink(linkpath) - link_metadata = hashutil.hash_data(data) - link_metadata.update({ - 'data': data, - 'length': len(data), - 'name': os.path.basename(linkpath), - 'perms': GitPerm.LINK, - 'type': GitType.BLOB, - 'path': linkpath - }) - - return link_metadata - - -def compute_blob_metadata(filepath): - """Given a filepath, compute the git metadata. - - Args: - filepath: absolute pathname of the file. - - Returns: - Dictionary of values: - - name: basename of the file - - perms: git permission for file - - type: git type for file - - """ - blob_metadata = hashutil.hash_path(filepath) - perms = GitPerm.EXEC if os.access(filepath, os.X_OK) else GitPerm.BLOB - blob_metadata.update({ - 'name': os.path.basename(filepath), - 'perms': perms, - 'type': GitType.BLOB, - 'path': filepath - }) - - return blob_metadata - - -def compute_tree_metadata(dirname, ls_hashes): - """Given a dirname, compute the git metadata. - - Args: - dirname: absolute pathname of the directory. - - Returns: - Dictionary of values: - - name: basename of the directory - - perms: git permission for directory - - type: git type for directory - - """ - return { - 'sha1_git': compute_directory_git_sha1(dirname, ls_hashes), - 'name': os.path.basename(dirname), - 'perms': GitPerm.TREE, - 'type': GitType.TREE, - 'path': dirname - } - - -def walk_and_compute_sha1_from_directory(rootdir): - """Compute git sha1 from directory rootdir. - - Returns: - Dictionary of entries with keys and as values a list of - directory entries. - Those are list of dictionary with keys: - - 'perms' - - 'type' - - 'name' - - 'sha1_git' - - and specifically content: 'sha1', 'sha256', ... - - Note: - One special key is ROOT_TREE_KEY to indicate the upper root of the - directory (this is the revision's directory). - - Raises: - Nothing - If something is raised, this is a programmatic error. - - """ - ls_hashes = {} - all_links = set() - - for dirpath, dirnames, filenames in os.walk(rootdir, topdown=False): - hashes = [] - - links = [os.path.join(dirpath, file) - for file in (filenames+dirnames) - if os.path.islink(os.path.join(dirpath, file))] - - for linkpath in links: - all_links.add(linkpath) - m_hashes = compute_link_metadata(linkpath) - hashes.append(m_hashes) - - only_files = [os.path.join(dirpath, file) - for file in filenames - if os.path.join(dirpath, file) not in all_links] - for filepath in only_files: - m_hashes = compute_blob_metadata(filepath) - hashes.append(m_hashes) - - ls_hashes[dirpath] = hashes - - dir_hashes = [] - subdirs = [os.path.join(dirpath, dir) - for dir in dirnames - if os.path.join(dirpath, dir) - not in all_links] - for fulldirname in subdirs: - tree_hash = compute_tree_metadata(fulldirname, ls_hashes) - dir_hashes.append(tree_hash) - - ls_hashes[dirpath].extend(dir_hashes) - - # compute the current directory hashes - root_hash = { - 'sha1_git': compute_directory_git_sha1(rootdir, ls_hashes), - 'path': rootdir, - 'name': os.path.basename(rootdir), - 'perms': GitPerm.TREE, - 'type': GitType.TREE - } - ls_hashes[ROOT_TREE_KEY] = [root_hash] - - return ls_hashes diff --git a/swh/loader/dir/loader.py b/swh/loader/dir/loader.py index 59b439b..57e7933 100644 --- a/swh/loader/dir/loader.py +++ b/swh/loader/dir/loader.py @@ -1,544 +1,198 @@ -# Copyright (C) 2015 The Software Heritage developers +# Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import logging import os -import psycopg2 -import requests import sys -import traceback import uuid -from retrying import retry - -from swh.core import config - -from swh.loader.dir import converters +from swh.loader.core import loader, converters from swh.model import git from swh.model.git import GitType -def send_in_packets(source_list, formatter, sender, packet_size, - packet_size_bytes=None, *args, **kwargs): - """Send objects from `source_list`, passed through `formatter` (with - extra args *args, **kwargs), using the `sender`, in packets of - `packet_size` objects (and of max `packet_size_bytes`). - - """ - formatted_objects = [] - count = 0 - if not packet_size_bytes: - packet_size_bytes = 0 - for obj in source_list: - formatted_object = formatter(obj, *args, **kwargs) - if formatted_object: - formatted_objects.append(formatted_object) - else: - continue - if packet_size_bytes: - count += formatted_object['length'] - if len(formatted_objects) >= packet_size or count > packet_size_bytes: - sender(formatted_objects) - formatted_objects = [] - count = 0 - - if formatted_objects: - sender(formatted_objects) - - -def retry_loading(error): - """Retry policy when the database raises an integrity error""" - exception_classes = [ - # raised when two parallel insertions insert the same data. - psycopg2.IntegrityError, - # raised when uWSGI restarts and hungs up on the worker. - requests.exceptions.ConnectionError, - ] - - if not any(isinstance(error, exc) for exc in exception_classes): - return False - - # FIXME: it could be DirLoaderWithHistory, TarLoader - logger = logging.getLogger('swh.loader.dir.DirLoader') - - error_name = error.__module__ + '.' + error.__class__.__name__ - logger.warning('Retry loading a batch', exc_info=False, extra={ - 'swh_type': 'storage_retry', - 'swh_exception_type': error_name, - 'swh_exception': traceback.format_exception( - error.__class__, - error, - error.__traceback__, - ), - }) - - return True - - -class DirLoader(config.SWHConfig): +class DirLoader(loader.SWHLoader): """A bulk loader for a directory. This will load the content of the directory. """ - DEFAULT_CONFIG = { - 'storage_class': ('str', 'remote_storage'), - 'storage_args': ('list[str]', ['http://localhost:5000/']), - - 'send_contents': ('bool', True), - 'send_directories': ('bool', True), - 'send_revisions': ('bool', True), - 'send_releases': ('bool', True), - 'send_occurrences': ('bool', True), - - 'content_packet_size': ('int', 10000), - 'content_packet_size_bytes': ('int', 1024 * 1024 * 1024), - 'directory_packet_size': ('int', 25000), - 'revision_packet_size': ('int', 100000), - 'release_packet_size': ('int', 100000), - 'occurrence_packet_size': ('int', 100000), - } - - def __init__(self, config): - self.config = config - - if self.config['storage_class'] == 'remote_storage': - from swh.storage.api.client import RemoteStorage as Storage - else: - from swh.storage import Storage - - self.storage = Storage(*self.config['storage_args']) - - self.log = logging.getLogger('swh.loader.dir.DirLoader') - - def open_fetch_history(self, origin_id): - return self.storage.fetch_history_start(origin_id) - - def close_fetch_history(self, fetch_history_id, res): - result = None - if 'objects' in res: - result = { - 'contents': len(res['objects'].get(GitType.BLOB, [])), - 'directories': len(res['objects'].get(GitType.TREE, [])), - 'revisions': len(res['objects'].get(GitType.COMM, [])), - 'releases': len(res['objects'].get(GitType.RELE, [])), - 'occurrences': len(res['objects'].get(GitType.REFS, [])), - } + CONFIG_BASE_FILENAME = 'loader/dir.ini' - data = { - 'status': res['status'], - 'result': result, - 'stderr': res.get('stderr') - } - return self.storage.fetch_history_end(fetch_history_id, data) - - @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) - def send_contents(self, content_list): - """Actually send properly formatted contents to the database""" - num_contents = len(content_list) - log_id = str(uuid.uuid4()) - self.log.debug("Sending %d contents" % num_contents, - extra={ - 'swh_type': 'storage_send_start', - 'swh_content_type': 'content', - 'swh_num': num_contents, - 'swh_id': log_id, - }) - self.storage.content_add(content_list) - self.log.debug("Done sending %d contents" % num_contents, - extra={ - 'swh_type': 'storage_send_end', - 'swh_content_type': 'content', - 'swh_num': num_contents, - 'swh_id': log_id, - }) - - @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) - def send_directories(self, directory_list): - """Actually send properly formatted directories to the database""" - num_directories = len(directory_list) - log_id = str(uuid.uuid4()) - self.log.debug("Sending %d directories" % num_directories, - extra={ - 'swh_type': 'storage_send_start', - 'swh_content_type': 'directory', - 'swh_num': num_directories, - 'swh_id': log_id, - }) - self.storage.directory_add(directory_list) - self.log.debug("Done sending %d directories" % num_directories, - extra={ - 'swh_type': 'storage_send_end', - 'swh_content_type': 'directory', - 'swh_num': num_directories, - 'swh_id': log_id, - }) - - @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) - def send_revisions(self, revision_list): - """Actually send properly formatted revisions to the database""" - num_revisions = len(revision_list) - log_id = str(uuid.uuid4()) - self.log.debug("Sending %d revisions" % num_revisions, - extra={ - 'swh_type': 'storage_send_start', - 'swh_content_type': 'revision', - 'swh_num': num_revisions, - 'swh_id': log_id, - }) - self.storage.revision_add(revision_list) - self.log.debug("Done sending %d revisions" % num_revisions, - extra={ - 'swh_type': 'storage_send_end', - 'swh_content_type': 'revision', - 'swh_num': num_revisions, - 'swh_id': log_id, - }) - - @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) - def send_releases(self, release_list): - """Actually send properly formatted releases to the database""" - num_releases = len(release_list) - log_id = str(uuid.uuid4()) - self.log.debug("Sending %d releases" % num_releases, - extra={ - 'swh_type': 'storage_send_start', - 'swh_content_type': 'release', - 'swh_num': num_releases, - 'swh_id': log_id, - }) - self.storage.release_add(release_list) - self.log.debug("Done sending %d releases" % num_releases, - extra={ - 'swh_type': 'storage_send_end', - 'swh_content_type': 'release', - 'swh_num': num_releases, - 'swh_id': log_id, - }) - - @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) - def send_occurrences(self, occurrence_list): - """Actually send properly formatted occurrences to the database""" - num_occurrences = len(occurrence_list) - log_id = str(uuid.uuid4()) - self.log.debug("Sending %d occurrences" % num_occurrences, - extra={ - 'swh_type': 'storage_send_start', - 'swh_content_type': 'occurrence', - 'swh_num': num_occurrences, - 'swh_id': log_id, - }) - self.storage.occurrence_add(occurrence_list) - self.log.debug("Done sending %d occurrences" % num_occurrences, - extra={ - 'swh_type': 'storage_send_end', - 'swh_content_type': 'occurrence', - 'swh_num': num_occurrences, - 'swh_id': log_id, - }) - - def bulk_send_blobs(self, objects, blobs, origin_id): - """Format blobs as swh contents and send them to the database""" - packet_size = self.config['content_packet_size'] - packet_size_bytes = self.config['content_packet_size_bytes'] - max_content_size = self.config['content_size_limit'] - - send_in_packets(blobs, converters.blob_to_content, - self.send_contents, packet_size, - packet_size_bytes=packet_size_bytes, - log=self.log, - max_content_size=max_content_size, - origin_id=origin_id) - - def bulk_send_trees(self, objects, trees): - """Format trees as swh directories and send them to the database""" - packet_size = self.config['directory_packet_size'] - - send_in_packets(trees, converters.tree_to_directory, - self.send_directories, packet_size, - objects=objects, - log=self.log) - - def bulk_send_commits(self, objects, commits): - """Format commits as swh revisions and send them to the database""" - packet_size = self.config['revision_packet_size'] - - send_in_packets(commits, (lambda x, objects={}, log=None: x), - self.send_revisions, packet_size, - objects=objects, - log=self.log) - - def bulk_send_annotated_tags(self, objects, tags): - """Format annotated tags (pygit2.Tag objects) as swh releases and send - them to the database - """ - packet_size = self.config['release_packet_size'] - - send_in_packets(tags, (lambda x, objects={}, log=None: x), - self.send_releases, packet_size, - log=self.log) - - def bulk_send_refs(self, objects, refs): - """Format git references as swh occurrences and send them to the - database - """ - packet_size = self.config['occurrence_packet_size'] - send_in_packets(refs, converters.ref_to_occurrence, - self.send_occurrences, packet_size) + def __init__(self, + origin_id, + logging_class='swh.loader.dir.DirLoader', + config=None): + super().__init__(origin_id, logging_class, config=config) def list_repo_objs(self, dir_path, revision, release): """List all objects from dir_path. Args: - dir_path (path): the directory to list - revision: revision dictionary representation - release: release dictionary representation Returns: a dict containing lists of `Oid`s with keys for each object type: - CONTENT - DIRECTORY """ def get_objects_per_object_type(objects_per_path): m = { GitType.BLOB: [], GitType.TREE: [], GitType.COMM: [], GitType.RELE: [] } for tree_path in objects_per_path: objs = objects_per_path[tree_path] for obj in objs: m[obj['type']].append(obj) return m def _revision_from(tree_hash, revision, objects): full_rev = dict(revision) full_rev['directory'] = tree_hash full_rev = converters.commit_to_revision(full_rev, objects) full_rev['id'] = git.compute_revision_sha1_git(full_rev) return full_rev def _release_from(revision_hash, release): full_rel = dict(release) full_rel['target'] = revision_hash full_rel['target_type'] = 'revision' full_rel = converters.annotated_tag_to_release(full_rel) full_rel['id'] = git.compute_release_sha1_git(full_rel) return full_rel log_id = str(uuid.uuid4()) sdir_path = dir_path.decode('utf-8') self.log.info("Started listing %s" % dir_path, extra={ 'swh_type': 'dir_list_objs_start', 'swh_repo': sdir_path, 'swh_id': log_id, }) objects_per_path = git.walk_and_compute_sha1_from_directory(dir_path) objects = get_objects_per_object_type(objects_per_path) tree_hash = objects_per_path[git.ROOT_TREE_KEY][0]['sha1_git'] full_rev = _revision_from(tree_hash, revision, objects_per_path) objects[GitType.COMM] = [full_rev] if release and 'name' in release: full_rel = _release_from(full_rev['id'], release) objects[GitType.RELE] = [full_rel] self.log.info("Done listing the objects in %s: %d contents, " "%d directories, %d revisions, %d releases" % ( sdir_path, len(objects[GitType.BLOB]), len(objects[GitType.TREE]), len(objects[GitType.COMM]), len(objects[GitType.RELE]) ), extra={ 'swh_type': 'dir_list_objs_end', 'swh_repo': sdir_path, 'swh_num_blobs': len(objects[GitType.BLOB]), 'swh_num_trees': len(objects[GitType.TREE]), 'swh_num_commits': len(objects[GitType.COMM]), 'swh_num_releases': len(objects[GitType.RELE]), 'swh_id': log_id, }) return objects, objects_per_path - def load_dir(self, dir_path, objects, objects_per_path, refs, origin_id): - if self.config['send_contents']: - self.bulk_send_blobs(objects_per_path, objects[GitType.BLOB], - origin_id) - else: - self.log.info('Not sending contents') - - if self.config['send_directories']: - self.bulk_send_trees(objects_per_path, objects[GitType.TREE]) - else: - self.log.info('Not sending directories') - - if self.config['send_revisions']: - self.bulk_send_commits(objects_per_path, objects[GitType.COMM]) - else: - self.log.info('Not sending revisions') - - if self.config['send_releases']: - self.bulk_send_annotated_tags(objects_per_path, - objects[GitType.RELE]) - else: - self.log.info('Not sending releases') - - if self.config['send_occurrences']: - self.bulk_send_refs(objects_per_path, refs) - else: - self.log.info('Not sending occurrences') - def process(self, dir_path, origin, revision, release, occurrences): """Load a directory in backend. Args: - dir_path: source of the directory to import - origin: Dictionary origin - id: origin's id - url: url origin we fetched - type: type of the origin - revision: Dictionary of information needed, keys are: - author_name: revision's author name - author_email: revision's author email - author_date: timestamp (e.g. 1444054085) - author_offset: date offset e.g. -0220, +0100 - committer_name: revision's committer name - committer_email: revision's committer email - committer_date: timestamp - committer_offset: date offset e.g. -0220, +0100 - type: type of revision dir, tar - message: synthetic message for the revision - release: Dictionary of information needed, keys are: - name: release name - date: release timestamp (e.g. 1444054085) - offset: release date offset e.g. -0220, +0100 - author_name: release author's name - author_email: release author's email - comment: release's comment message - occurrences: List of occurrences as dictionary. Information needed, keys are: - branch: occurrence's branch name - date: validity date (e.g. 2015-01-01 00:00:00+00) Returns: Dictionary with the following keys: - status: mandatory, the status result as a boolean - stderr: optional when status is True, mandatory otherwise - objects: the actual objects sent to swh storage """ def _occurrence_from(origin_id, revision_hash, occurrence): occ = dict(occurrence) occ.update({ 'target': revision_hash, 'target_type': 'revision', 'origin': origin_id, }) return occ def _occurrences_from(origin_id, revision_hash, occurrences): - full_occs = [] + occs = [] for occurrence in occurrences: - full_occ = _occurrence_from(origin_id, - revision_hash, - occurrence) - full_occs.append(full_occ) - return full_occs + occs.append(_occurrence_from(origin_id, + revision_hash, + occurrence)) + + return occs if not os.path.exists(dir_path): warn_msg = 'Skipping inexistant directory %s' % dir_path self.log.warn(warn_msg, extra={ 'swh_type': 'dir_repo_list_refs', 'swh_repo': dir_path, 'swh_num_refs': 0, }) return {'status': False, 'stderr': warn_msg} if isinstance(dir_path, str): dir_path = dir_path.encode(sys.getfilesystemencoding()) # to load the repository, walk all objects, compute their hash objects, objects_per_path = self.list_repo_objs(dir_path, revision, release) full_rev = objects[GitType.COMM][0] # only 1 revision - full_occs = _occurrences_from(origin['id'], - full_rev['id'], - occurrences) - - self.load_dir(dir_path, objects, objects_per_path, full_occs, - origin['id']) + # Update objects with release and occurrences + objects[GitType.RELE] = [full_rev] + objects[GitType.REFS] = _occurrences_from(origin['id'], + full_rev['id'], + occurrences) - objects[GitType.REFS] = full_occs + self.load(objects, objects_per_path) + self.flush() return {'status': True, 'objects': objects} - - -class DirLoaderWithHistory(DirLoader): - """A bulk loader for a directory. - - This will: - - create the origin if it does not exist - - open an entry in fetch_history - - load the content of the directory - - close the entry in fetch_history - - """ - def __init__(self, config): - super().__init__(config) - self.log = logging.getLogger('swh.loader.dir.DirLoaderWithHistory') - - def process(self, dir_path, origin, revision, release, occurrences): - """Load a directory in backend. - - Args: - - dir_path: source of the directory to import - - origin: Dictionary origin - - url: url origin we fetched - - type: type of the origin - - revision: Dictionary of information needed, keys are: - - author_name: revision's author name - - author_email: revision's author email - - author_date: timestamp (e.g. 1444054085) - - author_offset: date offset e.g. -0220, +0100 - - committer_name: revision's committer name - - committer_email: revision's committer email - - committer_date: timestamp - - committer_offset: date offset e.g. -0220, +0100 - - type: type of revision dir, tar - - message: synthetic message for the revision - - release: Dictionary of information needed, keys are: - - name: release name - - date: release timestamp (e.g. 1444054085) - - offset: release date offset e.g. -0220, +0100 - - author_name: release author's name - - author_email: release author's email - - comment: release's comment message - - occurrences: List of occurrence dictionary. - Information needed, keys are: - - branch: occurrence's branch name - - date: validity date (e.g. 2015-01-01 00:00:00+00) - - """ - origin['id'] = self.storage.origin_add_one(origin) - - fetch_history_id = self.open_fetch_history(origin['id']) - - result = super().process(dir_path, origin, revision, release, - occurrences) - - self.close_fetch_history(fetch_history_id, result) diff --git a/swh/loader/dir/tasks.py b/swh/loader/dir/tasks.py index 7f46076..a4dd591 100644 --- a/swh/loader/dir/tasks.py +++ b/swh/loader/dir/tasks.py @@ -1,35 +1,35 @@ -# Copyright (C) 2015 The Software Heritage developers +# Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from swh.scheduler.task import Task +from swh.loader.dir.loader import DirLoader +from swh.loader.core import tasks -from swh.loader.dir.loader import DirLoaderWithHistory - -class LoadDirRepository(Task): +class LoadDirRepository(tasks.LoaderCoreTask): """Import a directory to Software Heritage """ task_queue = 'swh_loader_dir' - CONFIG_BASE_FILENAME = 'loader/dir.ini' - ADDITIONAL_CONFIG = {} - - def __init__(self): - self.config = DirLoaderWithHistory.parse_config_file( - base_filename=self.CONFIG_BASE_FILENAME, - additional_configs=[self.ADDITIONAL_CONFIG], - ) - def run(self, dir_path, origin, revision, release, occurrences): """Import a directory. Args: cf. swh.loader.dir.loader.run docstring """ - loader = DirLoaderWithHistory(self.config) - loader.log = self.log - loader.process(dir_path, origin, revision, release, occurrences) + storage = DirLoader().storage + + origin['id'] = storage.origin_add_one(origin) + + fetch_history_id = self.open_fetch_history(storage, origin['id']) + + result = DirLoader(origin['id']).process(dir_path, + origin, + revision, + release, + occurrences) + + self.close_fetch_history(storage, fetch_history_id, result) diff --git a/swh/loader/dir/tests/test_converters.py b/swh/loader/dir/tests/test_converters.py deleted file mode 100644 index 0643953..0000000 --- a/swh/loader/dir/tests/test_converters.py +++ /dev/null @@ -1,337 +0,0 @@ -# Copyright (C) 2015 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import os -import shutil -import tempfile -import unittest - -from nose.tools import istest - -from swh.loader.dir import converters -from swh.model import git - - -def tmpfile_with_content(fromdir, contentfile): - """Create a temporary file with content contentfile in directory fromdir. - - """ - tmpfilepath = tempfile.mktemp( - suffix='.swh', - prefix='tmp-file-for-test', - dir=fromdir) - - with open(tmpfilepath, 'wb') as f: - f.write(contentfile) - - return tmpfilepath - - -class TestConverters(unittest.TestCase): - - @classmethod - def setUpClass(cls): - super().setUpClass() - cls.tmpdir = tempfile.mkdtemp(prefix='test-swh-loader-dir.') - - @classmethod - def tearDownClass(cls): - shutil.rmtree(cls.tmpdir) - super().tearDownClass() - - @istest - def format_to_minutes(self): - self.assertEquals(converters.format_to_minutes('+0100'), 60) - self.assertEquals(converters.format_to_minutes('-0200'), -120) - self.assertEquals(converters.format_to_minutes('+1250'), 12*60+50) - self.assertEquals(converters.format_to_minutes('+0000'), 0) - self.assertEquals(converters.format_to_minutes('-0000'), 0) - - @istest - def annotated_tag_to_release(self): - # given - release = { - 'id': '123', - 'target': '456', - 'target_type': 'revision', - 'name': 'some-release', - 'comment': 'some-comment-on-release', - 'date': 1444054085, - 'offset': '-0300', - 'author_name': 'someone', - 'author_email': 'someone@whatelse.eu', - } - - expected_release = { - 'target': '456', - 'target_type': 'revision', - 'name': b'some-release', - 'message': b'some-comment-on-release', - 'date': { - 'timestamp': 1444054085, - 'offset': -180 - }, - 'author': { - 'name': b'someone', - 'email': b'someone@whatelse.eu', - }, - 'synthetic': True, - } - - # when - actual_release = converters.annotated_tag_to_release(release) - - # then - self.assertDictEqual(actual_release, expected_release) - - @istest - def blob_to_content_visible_data(self): - # given - contentfile = b'temp file for testing blob to content conversion' - tmpfilepath = tmpfile_with_content(self.tmpdir, contentfile) - - obj = { - 'path': tmpfilepath, - 'perms': git.GitPerm.BLOB, - 'type': git.GitType.BLOB, - 'sha1': 'some-sha1', - 'sha256': 'some-sha256', - 'sha1_git': 'some-sha1git', - } - - expected_blob = { - 'data': contentfile, - 'length': len(contentfile), - 'status': 'visible', - 'sha1': 'some-sha1', - 'sha256': 'some-sha256', - 'sha1_git': 'some-sha1git', - 'perms': git.GitPerm.BLOB.value, - 'type': git.GitType.BLOB.value, - } - - # when - actual_blob = converters.blob_to_content(obj) - - # then - self.assertEqual(actual_blob, expected_blob) - - @istest - def blob_to_content_link(self): - # given - contentfile = b'temp file for testing blob to content conversion' - tmpfilepath = tmpfile_with_content(self.tmpdir, contentfile) - tmplinkpath = tempfile.mktemp(dir=self.tmpdir) - os.symlink(tmpfilepath, tmplinkpath) - - obj = { - 'path': tmplinkpath, - 'perms': git.GitPerm.BLOB, - 'type': git.GitType.BLOB, - 'sha1': 'some-sha1', - 'sha256': 'some-sha256', - 'sha1_git': 'some-sha1git', - } - - expected_blob = { - 'data': contentfile, - 'length': len(tmpfilepath), - 'status': 'visible', - 'sha1': 'some-sha1', - 'sha256': 'some-sha256', - 'sha1_git': 'some-sha1git', - 'perms': git.GitPerm.BLOB.value, - 'type': git.GitType.BLOB.value, - } - - # when - actual_blob = converters.blob_to_content(obj) - - # then - self.assertEqual(actual_blob, expected_blob) - - @istest - def blob_to_content_link_with_data_length_populated(self): - # given - tmplinkpath = tempfile.mktemp(dir=self.tmpdir) - obj = { - 'length': 10, # wrong for test purposes - 'data': 'something wrong', # again for test purposes - 'path': tmplinkpath, - 'perms': git.GitPerm.BLOB, - 'type': git.GitType.BLOB, - 'sha1': 'some-sha1', - 'sha256': 'some-sha256', - 'sha1_git': 'some-sha1git', - } - - expected_blob = { - 'length': 10, - 'data': 'something wrong', - 'status': 'visible', - 'sha1': 'some-sha1', - 'sha256': 'some-sha256', - 'sha1_git': 'some-sha1git', - 'perms': git.GitPerm.BLOB.value, - 'type': git.GitType.BLOB.value, - } - - # when - actual_blob = converters.blob_to_content(obj) - - # then - self.assertEqual(actual_blob, expected_blob) - - @istest - def blob_to_content2_absent_data(self): - # given - contentfile = b'temp file for testing blob to content conversion' - tmpfilepath = tmpfile_with_content(self.tmpdir, contentfile) - - obj = { - 'path': tmpfilepath, - 'perms': git.GitPerm.BLOB, - 'type': git.GitType.BLOB, - 'sha1': 'some-sha1', - 'sha256': 'some-sha256', - 'sha1_git': 'some-sha1git', - } - - expected_blob = { - 'length': len(contentfile), - 'status': 'absent', - 'sha1': 'some-sha1', - 'sha256': 'some-sha256', - 'sha1_git': 'some-sha1git', - 'perms': git.GitPerm.BLOB.value, - 'type': git.GitType.BLOB.value, - 'reason': 'Content too large', - 'origin': 190 - } - - # when - actual_blob = converters.blob_to_content(obj, None, - max_content_size=10, - origin_id=190) - - # then - self.assertEqual(actual_blob, expected_blob) - - @istest - def tree_to_directory_no_entries(self): - # given - tree = { - 'path': 'foo', - 'sha1_git': b'tree_sha1_git' - } - objects = { - 'foo': [{'type': git.GitType.TREE, - 'perms': git.GitPerm.TREE, - 'name': 'bar', - 'sha1_git': b'sha1-target'}, - {'type': git.GitType.BLOB, - 'perms': git.GitPerm.BLOB, - 'name': 'file-foo', - 'sha1_git': b'file-foo-sha1-target'}] - } - - expected_directory = { - 'id': b'tree_sha1_git', - 'entries': [{'type': 'dir', - 'perms': int(git.GitPerm.TREE.value), - 'name': 'bar', - 'target': b'sha1-target'}, - {'type': 'file', - 'perms': int(git.GitPerm.BLOB.value), - 'name': 'file-foo', - 'target': b'file-foo-sha1-target'}] - } - - # when - actual_directory = converters.tree_to_directory(tree, objects) - - # then - self.assertEqual(actual_directory, expected_directory) - - @istest - def commit_to_revision(self): - # given - commit = { - 'sha1_git': 'commit-git-sha1', - 'author_date': 1444054085, - 'author_offset': '+0000', - 'committer_date': 1444054085, - 'committer_offset': '-0000', - 'type': 'tar', - 'message': 'synthetic-message-input', - 'author_name': 'author-name', - 'author_email': 'author-email', - 'committer_name': 'committer-name', - 'committer_email': 'committer-email', - 'metadata': {'checksums': {'sha1': b'sha1-as-bytes'}}, - 'directory': 'targeted-tree-sha1', - } - - objects = { - git.ROOT_TREE_KEY: [{'sha1_git': 'targeted-tree-sha1'}] - } - - expected_revision = { - 'date': { - 'timestamp': 1444054085, - 'offset': 0, - }, - 'committer_date': { - 'timestamp': 1444054085, - 'offset': 0, - }, - 'type': 'tar', - 'directory': 'targeted-tree-sha1', - 'message': b'synthetic-message-input', - 'author': { - 'name': b'author-name', - 'email': b'author-email', - }, - 'committer': { - 'name': b'committer-name', - 'email': b'committer-email', - }, - 'synthetic': True, - 'metadata': {'checksums': {'sha1': b'sha1-as-bytes'}}, - 'parents': [], - } - - # when - actual_revision = converters.commit_to_revision(commit, objects) - - # then - self.assertEquals(actual_revision, expected_revision) - - @istest - def ref_to_occurrence_1(self): - # when - actual_occ = converters.ref_to_occurrence({ - 'id': 'some-id', - 'branch': 'some/branch' - }) - # then - self.assertEquals(actual_occ, { - 'id': 'some-id', - 'branch': b'some/branch' - }) - - @istest - def ref_to_occurrence_2(self): - # when - actual_occ = converters.ref_to_occurrence({ - 'id': 'some-id', - 'branch': b'some/branch' - }) - - # then - self.assertEquals(actual_occ, { - 'id': 'some-id', - 'branch': b'some/branch' - }) diff --git a/swh/loader/dir/tests/test_loader.py b/swh/loader/dir/tests/test_loader.py index 831d81d..7a0b59a 100644 --- a/swh/loader/dir/tests/test_loader.py +++ b/swh/loader/dir/tests/test_loader.py @@ -1,111 +1,125 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import shutil import subprocess import tempfile import unittest from nose.tools import istest from swh.loader.dir.loader import DirLoader from swh.model.git import GitType class TestLoader(unittest.TestCase): @classmethod def setUpClass(cls): super().setUpClass() cls.tmp_root_path = tempfile.mkdtemp().encode('utf-8') start_path = os.path.dirname(__file__).encode('utf-8') sample_folder_archive = os.path.join(start_path, b'../../../../..', b'swh-storage-testdata', b'dir-folders', b'sample-folder.tgz') cls.root_path = os.path.join(cls.tmp_root_path, b'sample-folder') # uncompress the sample folder subprocess.check_output( ['tar', 'xvf', sample_folder_archive, '-C', cls.tmp_root_path], ) @classmethod def tearDownClass(cls): super().tearDownClass() shutil.rmtree(cls.tmp_root_path) def setUp(self): super().setUp() self.info = { 'storage_class': 'remote_storage', 'storage_args': ['http://localhost:5000/'], + 'content_size_limit': 104857600, + 'log_db': 'dbname=softwareheritage-log', + 'directory_packet_size': 25000, + 'content_packet_size': 10000, + 'send_contents': True, + 'send_directories': True, + 'content_packet_size_bytes': 1073741824, + 'occurrence_packet_size': 100000, + 'send_revisions': True, + 'revision_packet_size': 100000, + 'content_packet_block_size_bytes': 104857600, + 'send_occurrences': True, + 'release_packet_size': 100000, + 'send_releases': True } self.origin = { 'url': 'file:///dev/null', 'type': 'dir', } self.occurrence = { 'branch': 'master', 'authority_id': 1, 'validity': '2015-01-01 00:00:00+00', } self.revision = { 'author_name': 'swh author', 'author_email': 'swh@inria.fr', 'author_date': '1444054085', 'author_offset': '+0200', 'committer_name': 'swh committer', 'committer_email': 'swh@inria.fr', 'committer_date': '1444054085', 'committer_offset': '+0200', 'type': 'tar', 'message': 'synthetic revision', 'metadata': {'foo': 'bar'}, } self.release = { 'name': 'v0.0.1', 'date': '1444054085', 'offset': '+0200', 'author_name': 'swh author', 'author_email': 'swh@inria.fr', 'comment': 'synthetic release', } - self.dirloader = DirLoader(self.info) + self.dirloader = DirLoader(origin_id=1, config=self.info) @istest def load_without_storage(self): # when objects, objects_per_path = self.dirloader.list_repo_objs( self.root_path, self.revision, self.release) # then self.assertEquals(len(objects), 4, "4 objects types, blob, tree, revision, release") self.assertEquals(len(objects[GitType.BLOB]), 8, "8 contents: 3 files + 5 links") self.assertEquals(len(objects[GitType.TREE]), 5, "5 directories: 4 subdirs + 1 empty + 1 main dir") self.assertEquals(len(objects[GitType.COMM]), 1, "synthetic revision") self.assertEquals(len(objects[GitType.RELE]), 1, "synthetic release") self.assertEquals(len(objects_per_path), 6, "5 folders + ") # print('objects: %s\n objects-per-path: %s\n' % # (objects.keys(), # objects_per_path.keys())) diff --git a/version.txt b/version.txt index 5ca3220..102b507 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.17-0-gc769213 \ No newline at end of file +v0.0.18-0-g704efe1 \ No newline at end of file