diff --git a/PKG-INFO b/PKG-INFO index a630e6b..4ae904a 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.core -Version: 0.0.36 +Version: 0.0.37 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/bin/swh-hashdir b/bin/swh-hashdir deleted file mode 100755 index f3f30cc..0000000 --- a/bin/swh-hashdir +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/python3 - -# Copyright (C) 2015 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import os -import sys - -from swh.core.hashutil import hashfile, hash_to_hex - -if __name__ == '__main__': - dirname = sys.argv[1] - - hashes = {} - for root, _dirs, files in os.walk(dirname): - for name in files: - path = os.path.join(root, name) - hashes[path] = { - algo: hash_to_hex(checksum) - for algo, checksum in hashfile(path).items() - } - hashes[path]['length'] = os.path.getsize(path) - - for (path, checksums) in hashes.items(): - print("\\\\x%(sha1)s\t\\\\x%(sha1_git)s\t\\\\x%(sha256)s\t%(length)d\tvisible" % # NOQA - checksums) diff --git a/bin/swh-hashfile b/bin/swh-hashfile deleted file mode 100755 index d7692aa..0000000 --- a/bin/swh-hashfile +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/python3 - -# Copyright (C) 2015 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import sys - -from swh.core.hashutil import hashfile, hash_to_hex - -if __name__ == '__main__': - fname = sys.argv[1] - for (algo, checksum) in sorted(hashfile(fname).items()): - print('%s\t%s' % (algo, hash_to_hex(checksum))) diff --git a/debian/changelog b/debian/changelog index 11f9596..4668c64 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,252 +1,253 @@ -swh-core (0.0.36-1~swh1~bpo9+1) stretch-swh; urgency=medium +swh-core (0.0.37-1~swh1) unstable-swh; urgency=medium - * Rebuild for stretch-backports. + * v0.0.37 + * Move test fixture in swh.core.tests.server_testing module - -- Antoine R. Dumont (@ardumont) Wed, 06 Dec 2017 12:03:29 +0100 + -- Antoine R. Dumont (@ardumont) Wed, 25 Apr 2018 15:00:02 +0200 swh-core (0.0.36-1~swh1) unstable-swh; urgency=medium * v0.0.36 * Migrate swh.loader.tar.tarball module in swh.core -- Antoine R. Dumont (@ardumont) Wed, 06 Dec 2017 12:03:29 +0100 swh-core (0.0.35-1~swh1) unstable-swh; urgency=medium * Release swh.core version 0.0.35 * Update packaging runes -- Nicolas Dandrimont Thu, 12 Oct 2017 18:07:50 +0200 swh-core (0.0.34-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.34 * New modular database test fixture -- Nicolas Dandrimont Mon, 07 Aug 2017 18:29:48 +0200 swh-core (0.0.33-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.33 * Be more conservative with remote API responses -- Nicolas Dandrimont Mon, 19 Jun 2017 19:01:38 +0200 swh-core (0.0.32-1~swh1) unstable-swh; urgency=medium * Release swh-core v0.0.32 * Add asynchronous streaming methods for internal APIs * Remove task arguments from systemd-journal loggers -- Nicolas Dandrimont Tue, 09 May 2017 14:04:22 +0200 swh-core (0.0.31-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.31 * Add explicit dependency on python3-systemd -- Nicolas Dandrimont Fri, 07 Apr 2017 15:11:26 +0200 swh-core (0.0.30-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.30 * drop swh.core.hashutil (moved to swh.model.hashutil) * add a systemd logger -- Nicolas Dandrimont Fri, 07 Apr 2017 11:49:15 +0200 swh-core (0.0.29-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.29 * Catch proper exception in the base API client -- Nicolas Dandrimont Thu, 02 Feb 2017 00:19:25 +0100 swh-core (0.0.28-1~swh1) unstable-swh; urgency=medium * v0.0.28 * Refactoring some common code into swh.core -- Antoine R. Dumont (@ardumont) Thu, 26 Jan 2017 14:54:22 +0100 swh-core (0.0.27-1~swh1) unstable-swh; urgency=medium * v0.0.27 * Fix issue with default boolean value -- Antoine R. Dumont (@ardumont) Thu, 20 Oct 2016 16:15:20 +0200 swh-core (0.0.26-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.26 * Raise an exception when a configuration file exists and is unreadable -- Nicolas Dandrimont Wed, 12 Oct 2016 10:16:09 +0200 swh-core (0.0.25-1~swh1) unstable-swh; urgency=medium * v0.0.25 * Add new function utils.cwd -- Antoine R. Dumont (@ardumont) Thu, 29 Sep 2016 21:29:37 +0200 swh-core (0.0.24-1~swh1) unstable-swh; urgency=medium * v0.0.24 * Deal with edge case in logger regarding json -- Antoine R. Dumont (@ardumont) Thu, 22 Sep 2016 12:21:09 +0200 swh-core (0.0.23-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.23 * Properly fix the PyYAML dependency -- Nicolas Dandrimont Tue, 23 Aug 2016 16:20:29 +0200 swh-core (0.0.22-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.22 * Proper loading of yaml and ini files in all paths -- Nicolas Dandrimont Fri, 19 Aug 2016 15:45:55 +0200 swh-core (0.0.21-1~swh1) unstable-swh; urgency=medium * v0.0.21 * Update test tools -- Antoine R. Dumont (@ardumont) Tue, 19 Jul 2016 14:47:01 +0200 swh-core (0.0.20-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.20 * Add some generic bytes <-> escaped unicode methods -- Nicolas Dandrimont Tue, 14 Jun 2016 16:54:41 +0200 swh-core (0.0.19-1~swh1) unstable-swh; urgency=medium * v0.0.19 * Resurrect swh.core.utils -- Antoine R. Dumont (@ardumont) Fri, 15 Apr 2016 12:40:43 +0200 swh-core (0.0.18-1~swh1) unstable-swh; urgency=medium * v0.0.18 * Add swh.core.utils * serializers: support UUIDs all around -- Antoine R. Dumont (@ardumont) Sat, 26 Mar 2016 11:16:33 +0100 swh-core (0.0.17-1~swh1) unstable-swh; urgency=medium * Release swh.core v0.0.17 * Allow serialization of UUIDs -- Nicolas Dandrimont Fri, 04 Mar 2016 11:40:56 +0100 swh-core (0.0.16-1~swh1) unstable-swh; urgency=medium * Release swh.core version 0.0.16 * add bytehex_to_hash and hash_to_bytehex in hashutil * move scheduling utilities to swh.scheduler -- Nicolas Dandrimont Fri, 19 Feb 2016 18:12:10 +0100 swh-core (0.0.15-1~swh1) unstable-swh; urgency=medium * Release v0.0.15 * Add hashutil.hash_git_object -- Nicolas Dandrimont Wed, 16 Dec 2015 16:31:26 +0100 swh-core (0.0.14-1~swh1) unstable-swh; urgency=medium * v0.0.14 * Add simple README * Update license * swh.core.hashutil.hashfile can now deal with filepath as bytes -- Antoine R. Dumont (@ardumont) Fri, 23 Oct 2015 11:13:14 +0200 swh-core (0.0.13-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.13 -- Nicolas Dandrimont Fri, 09 Oct 2015 17:32:49 +0200 swh-core (0.0.12-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.12 -- Nicolas Dandrimont Tue, 06 Oct 2015 17:34:34 +0200 swh-core (0.0.11-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.11 -- Nicolas Dandrimont Sat, 03 Oct 2015 15:57:03 +0200 swh-core (0.0.10-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.core v0.0.10 -- Nicolas Dandrimont Sat, 03 Oct 2015 12:28:52 +0200 swh-core (0.0.9-1~swh1) unstable-swh; urgency=medium * Prepare deploying swh.core v0.0.9 -- Nicolas Dandrimont Sat, 03 Oct 2015 11:36:55 +0200 swh-core (0.0.8-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.8 -- Nicolas Dandrimont Thu, 01 Oct 2015 12:31:44 +0200 swh-core (0.0.7-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.7 -- Nicolas Dandrimont Thu, 01 Oct 2015 11:29:04 +0200 swh-core (0.0.6-1~swh1) unstable-swh; urgency=medium * Prepare deployment of swh.core v0.0.6 -- Nicolas Dandrimont Tue, 29 Sep 2015 16:48:44 +0200 swh-core (0.0.5-1~swh1) unstable-swh; urgency=medium * Prepare v0.0.5 deployment -- Nicolas Dandrimont Tue, 29 Sep 2015 16:08:32 +0200 swh-core (0.0.4-1~swh1) unstable-swh; urgency=medium * Tagging swh.core 0.0.4 -- Nicolas Dandrimont Fri, 25 Sep 2015 15:41:26 +0200 swh-core (0.0.3-1~swh1) unstable-swh; urgency=medium * Tag swh.core v0.0.3 -- Nicolas Dandrimont Fri, 25 Sep 2015 11:07:10 +0200 swh-core (0.0.2-1~swh1) unstable-swh; urgency=medium * Deploy v0.0.2 -- Nicolas Dandrimont Wed, 23 Sep 2015 12:08:50 +0200 swh-core (0.0.1-1~swh1) unstable-swh; urgency=medium * Initial release * Tag v0.0.1 for deployment -- Nicolas Dandrimont Tue, 22 Sep 2015 14:52:26 +0200 diff --git a/setup.py b/setup.py index 5b6c3ef..8b2024c 100644 --- a/setup.py +++ b/setup.py @@ -1,30 +1,30 @@ #!/usr/bin/env python3 from setuptools import setup, find_packages def parse_requirements(): requirements = [] for reqf in ('requirements.txt', 'requirements-swh.txt'): with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue requirements.append(line) return requirements setup( name='swh.core', description='Software Heritage core utilities', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DCORE/', packages=find_packages(), - scripts=['bin/swh-hashdir', 'bin/swh-hashfile'], + scripts=[], install_requires=parse_requirements(), setup_requires=['vcversioner'], vcversioner={}, include_package_data=True, ) diff --git a/swh.core.egg-info/PKG-INFO b/swh.core.egg-info/PKG-INFO index a630e6b..4ae904a 100644 --- a/swh.core.egg-info/PKG-INFO +++ b/swh.core.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.core -Version: 0.0.36 +Version: 0.0.37 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh.core.egg-info/SOURCES.txt b/swh.core.egg-info/SOURCES.txt index fd3b0f2..0e8546e 100644 --- a/swh.core.egg-info/SOURCES.txt +++ b/swh.core.egg-info/SOURCES.txt @@ -1,44 +1,43 @@ .gitignore AUTHORS LICENSE MANIFEST.in Makefile README.md requirements-swh.txt requirements.txt setup.py version.txt -bin/swh-hashdir -bin/swh-hashfile debian/changelog debian/compat debian/control debian/copyright debian/rules debian/source/format docs/.gitignore docs/Makefile docs/conf.py docs/index.rst docs/_static/.placeholder docs/_templates/.placeholder sql/log-schema.sql swh/__init__.py swh.core.egg-info/PKG-INFO swh.core.egg-info/SOURCES.txt swh.core.egg-info/dependency_links.txt swh.core.egg-info/requires.txt swh.core.egg-info/top_level.txt swh/core/__init__.py swh/core/api.py swh/core/api_async.py swh/core/config.py swh/core/logger.py swh/core/serializers.py swh/core/tarball.py swh/core/utils.py swh/core/tests/db_testing.py +swh/core/tests/server_testing.py swh/core/tests/test_config.py swh/core/tests/test_logger.py swh/core/tests/test_serializers.py swh/core/tests/test_utils.py \ No newline at end of file diff --git a/swh/core/tests/server_testing.py b/swh/core/tests/server_testing.py new file mode 100644 index 0000000..e801a3e --- /dev/null +++ b/swh/core/tests/server_testing.py @@ -0,0 +1,146 @@ +# Copyright (C) 2015-2018 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import abc +import aiohttp +import multiprocessing +import socket +import time + +from urllib.request import urlopen + + +class ServerTestFixtureBaseClass(metaclass=abc.ABCMeta): + """Base class for http client/server testing implementations. + + Override this class to implement the following methods: + - process_config: to do something needed for the server + configuration (e.g propagate the configuration to other part) + - define_worker_function: define the function that will actually + run the server. + + To ensure test isolation, each test will run in a different server + and a different folder. + + In order to correctly work, the subclass must call the parents + class's setUp() and tearDown() methods. + + """ + def setUp(self): + super().setUp() + self.start_server() + + def tearDown(self): + self.stop_server() + super().tearDown() + + def url(self): + return 'http://127.0.0.1:%d/' % self.port + + def process_config(self): + """Process the server's configuration. Do something useful for + example, pass along the self.config dictionary inside the + self.app. + + By default, do nothing. + + """ + pass + + @abc.abstractmethod + def define_worker_function(self, app, port): + """Define how the actual implementation server will run. + + """ + pass + + def start_server(self): + """ Spawn the API server using multiprocessing. + """ + self.process = None + + self.process_config() + + # Get an available port number + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(('127.0.0.1', 0)) + self.port = sock.getsockname()[1] + sock.close() + + worker_fn = self.define_worker_function() + + self.process = multiprocessing.Process( + target=worker_fn, args=(self.app, self.port) + ) + self.process.start() + + # Wait max 5 seconds for server to spawn + i = 0 + while i < 500: + try: + urlopen(self.url()) + except Exception: + i += 1 + time.sleep(0.01) + else: + return + + def stop_server(self): + """ Terminate the API server's process. + """ + if self.process: + self.process.terminate() + + +class ServerTestFixture(ServerTestFixtureBaseClass): + """Base class for http client/server testing (e.g flask). + + Mix this in a test class in order to have access to an http server + running in background. + + Note that the subclass should define a dictionary in self.config + that contains the server config. And an application in self.app + that corresponds to the type of server the tested client needs. + + To ensure test isolation, each test will run in a different server + and a different folder. + + In order to correctly work, the subclass must call the parents + class's setUp() and tearDown() methods. + """ + def process_config(self): + # WSGI app configuration + for key, value in self.config.items(): + self.app.config[key] = value + + def define_worker_function(self): + def worker(app, port): + return app.run(port=port, use_reloader=False) + + return worker + + +class ServerTestFixtureAsync(ServerTestFixtureBaseClass): + """Base class for http client/server async testing (e.g aiohttp). + + Mix this in a test class in order to have access to an http server + running in background. + + Note that the subclass should define an application in self.app + that corresponds to the type of server the tested client needs. + + To ensure test isolation, each test will run in a different server + and a different folder. + + In order to correctly work, the subclass must call the parents + class's setUp() and tearDown() methods. + + """ + def define_worker_function(self): + def worker(app, port): + return aiohttp.web.run_app(app, port=int(port), + print=lambda *_: None) + + return worker diff --git a/version.txt b/version.txt index 64c0eae..6cf8d66 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.36-0-g141d1a3 \ No newline at end of file +v0.0.37-0-g5de410e \ No newline at end of file