diff --git a/debian/control b/debian/control index 868fad918..1b2682818 100644 --- a/debian/control +++ b/debian/control @@ -1,50 +1,50 @@ Source: swh-storage Maintainer: Software Heritage developers Section: python Priority: optional Build-Depends: debhelper (>= 9), dh-python (>= 2), python3-aiohttp, python3-all, python3-click, python3-dateutil, python3-flask, python3-kafka, python3-nose, python3-psycopg2, python3-requests, python3-setuptools, python3-sqlalchemy (>= 1.0), - python3-swh.core (>= 0.0.28~), + python3-swh.core (>= 0.0.37~), python3-swh.model (>= 0.0.18~), python3-swh.objstorage (>= 0.0.17~), python3-swh.scheduler (>= 0.0.14~), python3-vcversioner Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DSTO/ Package: python3-swh.storage Architecture: all Depends: python3-swh.core (>= 0.0.28~), python3-swh.model (>= 0.0.18~), python3-swh.objstorage (>= 0.0.17~), ${misc:Depends}, ${python3:Depends} Description: Software Heritage storage utilities Package: python3-swh.storage.listener Architecture: all Depends: python3-kafka (>= 1.3.1~), python3-swh.journal (>= 0.0.2~), python3-swh.storage (= ${binary:Version}), ${misc:Depends}, ${python3:Depends} Description: Software Heritage storage listener Package: python3-swh.storage.schemata Architecture: all Depends: python3-sqlalchemy (>= 1.0), python3-swh.storage (= ${binary:Version}), ${misc:Depends}, ${python3:Depends} Description: Ancillary schemata for Software Heritage diff --git a/requirements-swh.txt b/requirements-swh.txt index f6eca736b..85ad7ed4c 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,4 +1,4 @@ -swh.core >= 0.0.28 +swh.core >= 0.0.37 swh.model >= 0.0.18 swh.objstorage >= 0.0.17 swh.scheduler >= 0.0.14 diff --git a/swh/storage/tests/server_testing.py b/swh/storage/tests/server_testing.py deleted file mode 100644 index f3c57588a..000000000 --- a/swh/storage/tests/server_testing.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright (C) 2015-2017 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import abc -import aiohttp -import multiprocessing -import socket -import time - -from urllib.request import urlopen - - -class ServerTestFixtureBaseClass(metaclass=abc.ABCMeta): - """Base class for http client/server testing implementations. - - Override this class to implement the following methods: - - process_config: to do something needed for the server - configuration (e.g propagate the configuration to other part) - - define_worker_function: define the function that will actually - run the server. - - To ensure test isolation, each test will run in a different server - and a different folder. - - In order to correctly work, the subclass must call the parents - class's setUp() and tearDown() methods. - - """ - def setUp(self): - super().setUp() - self.start_server() - - def tearDown(self): - self.stop_server() - super().tearDown() - - def url(self): - return 'http://127.0.0.1:%d/' % self.port - - def process_config(self): - """Process the server's configuration. Do something useful for - example, pass along the self.config dictionary inside the - self.app. - - By default, do nothing. - - """ - pass - - @abc.abstractmethod - def define_worker_function(self, app, port): - """Define how the actual implementation server will run. - - """ - pass - - def start_server(self): - """ Spawn the API server using multiprocessing. - """ - self.process = None - - self.process_config() - - # Get an available port number - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.bind(('127.0.0.1', 0)) - self.port = sock.getsockname()[1] - sock.close() - - worker_fn = self.define_worker_function() - - self.process = multiprocessing.Process( - target=worker_fn, args=(self.app, self.port) - ) - self.process.start() - - # Wait max 5 seconds for server to spawn - i = 0 - while i < 500: - try: - urlopen(self.url()) - except Exception: - i += 1 - time.sleep(0.01) - else: - return - - def stop_server(self): - """ Terminate the API server's process. - """ - if self.process: - self.process.terminate() - - -class ServerTestFixture(ServerTestFixtureBaseClass): - """Base class for http client/server testing (e.g flask). - - Mix this in a test class in order to have access to an http server - running in background. - - Note that the subclass should define a dictionary in self.config - that contains the server config. And an application in self.app - that corresponds to the type of server the tested client needs. - - To ensure test isolation, each test will run in a different server - and a different folder. - - In order to correctly work, the subclass must call the parents - class's setUp() and tearDown() methods. - """ - def process_config(self): - # WSGI app configuration - for key, value in self.config.items(): - self.app.config[key] = value - - def define_worker_function(self): - def worker(app, port): - return app.run(port=port, use_reloader=False) - - return worker - - -class ServerTestFixtureAsync(ServerTestFixtureBaseClass): - """Base class for http client/server async testing (e.g aiohttp). - - Mix this in a test class in order to have access to an http server - running in background. - - Note that the subclass should define an application in self.app - that corresponds to the type of server the tested client needs. - - To ensure test isolation, each test will run in a different server - and a different folder. - - In order to correctly work, the subclass must call the parents - class's setUp() and tearDown() methods. - - """ - def define_worker_function(self): - def worker(app, port): - return aiohttp.web.run_app(app, port=int(port), - print=lambda *_: None) - - return worker diff --git a/swh/storage/tests/test_api_client.py b/swh/storage/tests/test_api_client.py index 9e2d43489..6c1b7ba18 100644 --- a/swh/storage/tests/test_api_client.py +++ b/swh/storage/tests/test_api_client.py @@ -1,49 +1,49 @@ -# Copyright (C) 2015-2016 The Software Heritage developers +# Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest import tempfile +from swh.core.tests.server_testing import ServerTestFixture from swh.storage.tests.test_storage import CommonTestStorage -from swh.storage.tests.server_testing import ServerTestFixture from swh.storage.api.client import RemoteStorage from swh.storage.api.server import app class TestRemoteStorage(CommonTestStorage, ServerTestFixture, unittest.TestCase): """Test the remote storage API. This class doesn't define any tests as we want identical functionality between local and remote storage. All the tests are therefore defined in CommonTestStorage. """ def setUp(self): # ServerTestFixture needs to have self.objroot for # setUp() method, but this field is defined in # AbstractTestStorage's setUp() # To avoid confusion, override the self.objroot to a # one chosen in this class. storage_base = tempfile.mkdtemp() self.config = { 'storage': { 'cls': 'local', 'args': { 'db': 'dbname=%s' % self.TEST_STORAGE_DB_NAME, 'objstorage': { 'cls': 'pathslicing', 'args': { 'root': storage_base, 'slicing': '0:2/2:4/4:6', }, }, } } } self.app = app super().setUp() self.storage = RemoteStorage(self.url()) self.objroot = storage_base