$ gbp import-orig --uscan gbp:info: Launching uscan... gbp:info: Using uscan downloaded tarball ../mirakuru_2.3.0.orig.tar.gz What is the upstream version? [2.3.0] gbp:info: Importing '../mirakuru_2.3.0.orig.tar.gz' to branch 'debian/upstream'... gbp:info: Source package is mirakuru gbp:info: Upstream version is 2.3.0 gbp:info: Replacing upstream source on 'debian/unstable-swh' gbp:info: Successfully imported version 2.3.0 of ../mirakuru_2.3.0.orig.tar.gz $ gbp buildpackage --git-builder='sbuild -As' gbp:info: Tarballs 'mirakuru_2.1.0.orig.tar.gz' not found at '/home/tony/debian/tarballs/' gbp:info: Exporting 'HEAD' to '/home/tony/debian/build-area/mirakuru-tmp' gbp:info: Moving '/home/tony/debian/build-area/mirakuru-tmp' to '/home/tony/debian/build-area/mirakuru-2.1.0' gbp:info: Performing the build dh clean --with python3 --buildsystem=pybuild dh_auto_clean -O--buildsystem=pybuild I: pybuild base:217: python3.7 setup.py clean running clean removing '/home/tony/debian/build-area/mirakuru-2.1.0/.pybuild/cpython3_3.7_mirakuru/build' (and everything under it) 'build/bdist.linux-amd64' does not exist -- can't clean it 'build/scripts-3.7' does not exist -- can't clean it dh_autoreconf_clean -O--buildsystem=pybuild dh_clean -O--buildsystem=pybuild dpkg-source: info: using source format '3.0 (quilt)' dpkg-source: info: building mirakuru using existing ./mirakuru_2.1.0.orig.tar.gz dpkg-source: info: using patch list from debian/patches/series dpkg-source: error: cannot represent change to logo.png: binary file contents changed dpkg-source: error: add logo.png in debian/source/include-binaries if you want to store the modified binary in the debian tarball dpkg-source: warning: file mirakuru-2.1.0/requirements-lint.txt has no final newline (either original or modified version) dpkg-source: error: unrepresentable changes to source E: Failed to package source directory /home/tony/debian/build-area/mirakuru-2.1.0 gbp:error: 'sbuild -As' failed: it exited with 1 $ em debian/source/include-binaries + /nix/store/5j7hnf6yh6c48dqdfk09yarikm4hgygb-emacs-27.1/bin/emacsclient --create-frame -nw debian/source/include-binaries $ gbp buildpackage --git-builder='sbuild -As' gbp:info: Tarballs 'mirakuru_2.1.0.orig.tar.gz' not found at '/home/tony/debian/tarballs/' gbp:info: Exporting 'HEAD' to '/home/tony/debian/build-area/mirakuru-tmp' gbp:info: Moving '/home/tony/debian/build-area/mirakuru-tmp' to '/home/tony/debian/build-area/mirakuru-2.1.0' gbp:info: Performing the build dh clean --with python3 --buildsystem=pybuild dh_auto_clean -O--buildsystem=pybuild I: pybuild base:217: python3.7 setup.py clean running clean removing '/home/tony/debian/build-area/mirakuru-2.1.0/.pybuild/cpython3_3.7_mirakuru/build' (and everything under it) 'build/bdist.linux-amd64' does not exist -- can't clean it 'build/scripts-3.7' does not exist -- can't clean it dh_autoreconf_clean -O--buildsystem=pybuild dh_clean -O--buildsystem=pybuild dpkg-source: info: using source format '3.0 (quilt)' dpkg-source: info: building mirakuru using existing ./mirakuru_2.1.0.orig.tar.gz dpkg-source: info: using patch list from debian/patches/series dpkg-source: warning: file mirakuru-2.1.0/requirements-lint.txt has no final newline (either original or modified version) dpkg-source: info: local changes detected, the modified files are: mirakuru-2.1.0/.bumpversion.cfg mirakuru-2.1.0/.travis.yml mirakuru-2.1.0/AUTHORS.rst mirakuru-2.1.0/CHANGES.rst mirakuru-2.1.0/README.rst mirakuru-2.1.0/logo.svg mirakuru-2.1.0/mypy.ini mirakuru-2.1.0/requirements-lint.txt mirakuru-2.1.0/requirements-test.txt mirakuru-2.1.0/setup.py mirakuru-2.1.0/src/mirakuru/__init__.py mirakuru-2.1.0/src/mirakuru/base.py mirakuru-2.1.0/src/mirakuru/base_env.py mirakuru-2.1.0/src/mirakuru/exceptions.py mirakuru-2.1.0/src/mirakuru/http.py mirakuru-2.1.0/src/mirakuru/output.py mirakuru-2.1.0/tests/executors/test_executor.py mirakuru-2.1.0/tests/executors/test_executor_kill.py mirakuru-2.1.0/tests/executors/test_tcp_executor.py dpkg-source: info: you can integrate the local changes with dpkg-source --commit dpkg-source: error: aborting due to unexpected upstream changes, see /tmp/mirakuru_2.1.0-2~~swh1.diff.ei5huP E: Failed to package source directory /home/tony/debian/build-area/mirakuru-2.1.0 gbp:error: 'sbuild -As' failed: it exited with 1 $ cat /tmp/mirakuru_2.1.0-2~~swh1.diff.ei5huP Description: TODO: Put a short summary on the line above and replace this paragraph with a longer explanation of this change. Complete the meta-information with other relevant fields (see below for details). To make it easier, the information below has been extracted from the changelog. Adjust it or drop it. . mirakuru (2.1.0-2~~swh1) unstable-swh; urgency=medium . * Remove .coverage from installed directory Author: Nicolas Dandrimont --- The information above should follow the Patch Tagging Guidelines, please checkout http://dep.debian.net/deps/dep3/ to learn about the format. Here are templates for supplementary fields that you might want to add: Origin: , Bug: Bug-Debian: https://bugs.debian.org/ Bug-Ubuntu: https://launchpad.net/bugs/ Forwarded: Reviewed-By: Last-Update: 2021-03-17 --- mirakuru-2.1.0.orig/.bumpversion.cfg +++ mirakuru-2.1.0/.bumpversion.cfg @@ -2,7 +2,7 @@ commit = True tag = True message = "Release {new_version}" -current_version = 2.1.0 +current_version = 2.3.0 [bumpversion:file:setup.py] search = version='{current_version}' --- mirakuru-2.1.0.orig/.travis.yml +++ mirakuru-2.1.0/.travis.yml @@ -2,9 +2,10 @@ dist: xenial language: python conditions: v1 python: -- 3.6 +- 3.8 - 3.7 -- 3.8-dev +- 3.6 +- nightly - pypy3 # blocklist branches branches: @@ -23,7 +24,6 @@ after_success: jobs: include: - stage: linters - python: 3.7 install: - pip install -r requirements-lint.txt script: @@ -43,7 +43,6 @@ jobs: script: - pytest - stage: deploy - python: 3.7 if: tag IS present script: skip deploy: --- mirakuru-2.1.0.orig/AUTHORS.rst +++ mirakuru-2.1.0/AUTHORS.rst @@ -12,5 +12,6 @@ mirakuru along its history. * Daniel O'Connell * Michał Pawłowski * Grégoire Détrez +* Lars Gohr Great thanks to `Mateusz Lenik `_ for original package! --- mirakuru-2.1.0.orig/CHANGES.rst +++ mirakuru-2.1.0/CHANGES.rst @@ -1,6 +1,31 @@ CHANGELOG ========= +2.3.0 +---------- + +- [enhancement] Ability to set up expected exit code for executor. In Java exit codes 1- 127 have + special meaning, and the regular exit codes are offset by those of special meaning. + +2.2.0 +---------- + +- [enhancement] If process is being closed and the shutdown won't be clean (won't return exit code 0) + mirakuru will now rise ProcessFinishedWithError exception with exit_code + +2.1.2 +---------- + +- [bugfix][macos] Fixed typing issue on macOS + +2.1.1 +---------- + +- [bug] Always close connection for HTTPExecutor after_start_check +- [enhancement] Log debug message if execption occured during + HTTPExecutor start check +- [ehnancement] adjust typing handling in HTTPExecutor + 2.1.0 ---------- --- mirakuru-2.1.0.orig/README.rst +++ mirakuru-2.1.0/README.rst @@ -1,3 +1,6 @@ +.. image:: https://raw.githubusercontent.com/ClearcodeHQ/mirakuru/master/logo.png + :height: 100px + mirakuru ======== @@ -12,8 +15,8 @@ This is where you should consider using :target: https://pypi.python.org/pypi/mirakuru/ :alt: Latest PyPI version -.. image:: https://readthedocs.org/projects/mirakuru/badge/?version=v2.1.0 - :target: http://mirakuru.readthedocs.io/en/v2.1.0/ +.. image:: https://readthedocs.org/projects/mirakuru/badge/?version=v2.3.0 + :target: http://mirakuru.readthedocs.io/en/v2.3.0/ :alt: Documentation Status .. image:: https://img.shields.io/pypi/wheel/mirakuru.svg @@ -31,18 +34,14 @@ This is where you should consider using Package status -------------- -.. image:: https://travis-ci.org/ClearcodeHQ/mirakuru.svg?branch=v2.1.0 +.. image:: https://travis-ci.org/ClearcodeHQ/mirakuru.svg?branch=v2.3.0 :target: https://travis-ci.org/ClearcodeHQ/mirakuru :alt: Tests -.. image:: https://coveralls.io/repos/ClearcodeHQ/mirakuru/badge.png?branch=v2.1.0 - :target: https://coveralls.io/r/ClearcodeHQ/mirakuru?branch=v2.1.0 +.. image:: https://coveralls.io/repos/ClearcodeHQ/mirakuru/badge.png?branch=v2.3.0 + :target: https://coveralls.io/r/ClearcodeHQ/mirakuru?branch=v2.3.0 :alt: Coverage Status -.. image:: https://requires.io/github/ClearcodeHQ/mirakuru/requirements.svg?tag=v2.1.0 - :target: https://requires.io/github/ClearcodeHQ/mirakuru/requirements/?tag=v2.1.0 - :alt: Requirements Status - About ----- --- /dev/null +++ mirakuru-2.1.0/logo.svg @@ -0,0 +1,100 @@ + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + MIRAKURU + --- mirakuru-2.1.0.orig/mypy.ini +++ mirakuru-2.1.0/mypy.ini @@ -1,5 +1,6 @@ [mypy] check_untyped_defs = True +show_error_codes = True mypy_path = src [mypy-mirakuru.*] --- mirakuru-2.1.0.orig/requirements-lint.txt +++ mirakuru-2.1.0/requirements-lint.txt @@ -1,8 +1,8 @@ # linters pycodestyle==2.5.0 -pydocstyle==4.0.1 -pylint==2.3.1 +pydocstyle==5.0.2 +pylint==2.5.2 pygments restructuredtext-lint==1.3.0 -mypy==0.720 +mypy==0.770 -r requirements-test.txt \ No newline at end of file --- mirakuru-2.1.0.orig/requirements-test.txt +++ mirakuru-2.1.0/requirements-test.txt @@ -1,8 +1,8 @@ # test runs requirements (versions we'll be testing against) - automatically updated -psutil==5.6.3 -pytest==5.1.2 # tests framework used -pytest-cov==2.7.1 # coverage reports to verify tests quality -coverage==4.5.4 # pytest-cov -python-daemon==2.2.3 # used in test for easy creation of daemons +psutil==5.7.0 +pytest==5.4.2 # tests framework used +pytest-cov==2.8.1 # coverage reports to verify tests quality +coverage==5.1 # pytest-cov +python-daemon==2.2.4 # used in test for easy creation of daemons docutils # needed for python-daemon -e .[tests] --- mirakuru-2.1.0.orig/setup.py +++ mirakuru-2.1.0/setup.py @@ -53,7 +53,7 @@ def read(fname): setup( name='mirakuru', - version='2.1.0', + version='2.3.0', description='Process executor for tests.', long_description=( read('README.rst') + '\n\n' + read('CHANGES.rst') @@ -76,6 +76,7 @@ setup( 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Software Development :: Testing', --- mirakuru-2.1.0.orig/src/mirakuru/__init__.py +++ mirakuru-2.1.0/src/mirakuru/__init__.py @@ -33,7 +33,7 @@ from mirakuru.exceptions import ( ProcessExitedWithError, ) -__version__ = '2.1.0' +__version__ = '2.3.0' __all__ = ( 'Executor', --- mirakuru-2.1.0.orig/src/mirakuru/base.py +++ mirakuru-2.1.0/src/mirakuru/base.py @@ -38,13 +38,13 @@ from mirakuru.base_env import processes_ from mirakuru.exceptions import ( AlreadyRunning, ProcessExitedWithError, + ProcessFinishedWithError, TimeoutExpired, ) from mirakuru.compat import SIGKILL log = logging.getLogger(__name__) # pylint: disable=invalid-name - ENV_UUID = 'mirakuru_uuid' """ Name of the environment variable used by mirakuru to mark its subprocesses. @@ -63,13 +63,14 @@ ExecutorType = TypeVar("ExecutorType", b @atexit.register def cleanup_subprocesses() -> None: """On python exit: find possibly running subprocesses and kill them.""" - # pylint: disable=redefined-outer-name, reimported + # pylint: disable=redefined-outer-name, reimported, import-outside-toplevel # atexit functions tends to loose global imports sometimes so reimport # everything what is needed again here: import os import errno from mirakuru.base_env import processes_with_env from mirakuru.compat import SIGKILL + # pylint: enable=redefined-outer-name, reimported, import-outside-toplevel pids = processes_with_env(ENV_UUID, str(os.getpid())) for pid in pids: @@ -92,6 +93,7 @@ class SimpleExecutor: # pylint:disable= sleep: float = 0.1, sig_stop: int = signal.SIGTERM, sig_kill: int = SIGKILL, + exp_sig: int = None, envvars: Optional[Dict[str, str]] = None, stdin: Union[None, int, IO[Any]] = subprocess.PIPE, stdout: Union[None, int, IO[Any]] = subprocess.PIPE, @@ -111,6 +113,8 @@ class SimpleExecutor: # pylint:disable= default is `signal.SIGTERM` :param int sig_kill: signal used to kill process run by the executor. default is `signal.SIGKILL` (`signal.SIGTERM` on Windows) + :param int exp_sig: expected exit code. + default is None as it will fall back to the stop signal :param dict envvars: Additional environment variables :param int stdin: file descriptor for stdin :param int stdout: file descriptor for stdout @@ -145,6 +149,7 @@ class SimpleExecutor: # pylint:disable= self._sleep = sleep self._sig_stop = sig_stop self._sig_kill = sig_kill + self._exp_sig = exp_sig self._envvars = envvars or {} self._stdin = stdin @@ -294,7 +299,11 @@ class SimpleExecutor: # pylint:disable= log.debug("Killed process %d.", pid) return pids - def stop(self: SimpleExecutorType, sig: int = None) -> SimpleExecutorType: + def stop( + self: SimpleExecutorType, + sig: int = None, + exp_sig: int = None + ) -> SimpleExecutorType: """ Stop process running. @@ -302,6 +311,8 @@ class SimpleExecutor: # pylint:disable= :param int sig: signal used to stop process run by executor. None for default. + :param int exp_sig: expected exit code. + None for default. :returns: itself :rtype: SimpleExecutor @@ -316,6 +327,9 @@ class SimpleExecutor: # pylint:disable= if sig is None: sig = self._sig_stop + if exp_sig is None: + exp_sig = self._exp_sig + try: os.killpg(self.process.pid, sig) except OSError as err: @@ -335,8 +349,25 @@ class SimpleExecutor: # pylint:disable= # at this moment, process got killed, pass + if self.process is None: + # the process has already been force killed and cleaned up by the + # `wait_for` above. + return self self._kill_all_kids(sig) + exit_code = self.process.wait() self._clear_process() + + # Did the process shut down cleanly? A an exit code of `-sig` means + # that it has terminated due to signal `sig`, which is intended. So + # don't treat that as an error. + # pylint: disable=invalid-unary-operand-type + expected_exit_code = -sig + if exp_sig is not None: + expected_exit_code = -exp_sig + + if exit_code and exit_code != expected_exit_code: + raise ProcessFinishedWithError(self, exit_code) + return self @contextmanager --- mirakuru-2.1.0.orig/src/mirakuru/base_env.py +++ mirakuru-2.1.0/src/mirakuru/base_env.py @@ -21,7 +21,7 @@ import errno import logging import re import subprocess -from typing import Set +from typing import Set, List try: import psutil @@ -79,7 +79,7 @@ def processes_with_env_ps(env_name: str, :rtype: set """ pids: Set[int] = set() - ps_xe = '' + ps_xe: List[bytes] = [] try: cmd = 'ps', 'xe', '-o', 'pid,cmd' ps_xe = subprocess.check_output(cmd).splitlines() @@ -96,9 +96,9 @@ def processes_with_env_ps(env_name: str, env = f'{env_name}={env_value}' for line in ps_xe: - line = str(line) - if env in line: - match = PS_XE_PID_MATCH.match(line) + sline = str(line) + if env in sline: + match = PS_XE_PID_MATCH.match(sline) # This always matches: all lines other than the header (not # containing our environment variable) have a PID required by the # reggex. Still check it for mypy. --- mirakuru-2.1.0.orig/src/mirakuru/exceptions.py +++ mirakuru-2.1.0/src/mirakuru/exceptions.py @@ -101,3 +101,12 @@ class ProcessExitedWithError(ExecutorErr """ return (f"The process invoked by the {self.executor} executor has " f"exited with a non-zero code: {self.exit_code}.") + + +class ProcessFinishedWithError(ProcessExitedWithError): + """ + Raised when the process invoked by the executor fails when stopping. + + When a process is stopped, it should shut down cleanly and return zero as + exit code. When is returns a non-zero exit code, this exception is raised. + """ --- mirakuru-2.1.0.orig/src/mirakuru/http.py +++ mirakuru-2.1.0/src/mirakuru/http.py @@ -19,12 +19,15 @@ import re import socket +from logging import getLogger from urllib.parse import urlparse, urlencode from http.client import HTTPConnection, HTTPException from typing import Union, List, Tuple, Optional, Dict, Any from mirakuru.tcp import TCPExecutor +LOG = getLogger(__name__) + class HTTPExecutor(TCPExecutor): """Http enabled process executor.""" @@ -74,6 +77,9 @@ class HTTPExecutor(TCPExecutor): It'll be used to check process status on. """ + if not self.url.hostname: + raise ValueError("Url provided does not contain hostname") + port = self.url.port if port is None: port = self.DEFAULT_PORT @@ -89,9 +95,9 @@ class HTTPExecutor(TCPExecutor): ) def after_start_check(self) -> bool: - """Check if defined URL returns expected status to a HEAD request.""" + """Check if defined URL returns expected status to a check request.""" + conn = HTTPConnection(self.host, self.port) try: - conn = HTTPConnection(self.host, self.port) body = urlencode(self.payload) if self.payload else None headers = self.headers if self.headers else {} conn.request( @@ -100,12 +106,18 @@ class HTTPExecutor(TCPExecutor): body, headers, ) - status = str(conn.getresponse().status) + try: + status = str(conn.getresponse().status) + finally: + conn.close() if status == self.status or self.status_re.match(status): - conn.close() return True return False - except (HTTPException, socket.timeout, socket.error): + except (HTTPException, socket.timeout, socket.error) as ex: + LOG.debug( + "Encounter %s while trying to check if service has started.", + ex + ) return False --- mirakuru-2.1.0.orig/src/mirakuru/output.py +++ mirakuru-2.1.0/src/mirakuru/output.py @@ -133,7 +133,7 @@ class OutputExecutor(SimpleExecutor): return True return False - def _wait_for_output(self, *polls: Tuple[select.poll, IO[Any]]) -> bool: + def _wait_for_output(self, *polls: Tuple['select.poll', IO[Any]]) -> bool: """ Check if output matches banner. --- mirakuru-2.1.0.orig/tests/executors/test_executor.py +++ mirakuru-2.1.0/tests/executors/test_executor.py @@ -59,6 +59,22 @@ def test_stop_custom_signal_stop(): assert executor.running() is False +def test_stop_custom_exit_signal_stop(): + """Start process and expect it to finish with custom signal.""" + executor = SimpleExecutor('false', shell=True) + executor.start() + # false exits instant, so there should not be a process to stop + executor.stop(sig=signal.SIGQUIT, exp_sig=3) + assert executor.running() is False + + +def test_stop_custom_exit_signal_context(): + """Start process and expect custom exit signal in context manager.""" + with SimpleExecutor('false', exp_sig=3, shell=True) as executor: + executor.stop(sig=signal.SIGQUIT) + assert executor.running() is False + + def test_running_context(): """Start process and shuts it down.""" executor = SimpleExecutor(SLEEP_300) @@ -229,14 +245,14 @@ def test_executor_methods_returning_self def test_mirakuru_cleanup(): """Test if cleanup_subprocesses is fired correctly on python exit.""" cmd = f''' - python3 -c 'from mirakuru import SimpleExecutor; - from time import sleep; - import gc; - gc.disable(); - ex = SimpleExecutor( - ("python3", "{SAMPLE_DAEMON_PATH}")).start(); - sleep(1); - ' + python -c 'from mirakuru import SimpleExecutor; + from time import sleep; + import gc; + gc.disable(); + ex = SimpleExecutor( + ("python", "{SAMPLE_DAEMON_PATH}")).start(); + sleep(1); + ' ''' check_output(shlex.split(cmd.replace('\n', ''))) assert SAMPLE_DAEMON_PATH not in ps_aux() --- mirakuru-2.1.0.orig/tests/executors/test_executor_kill.py +++ mirakuru-2.1.0/tests/executors/test_executor_kill.py @@ -8,10 +8,12 @@ import errno import os from unittest.mock import patch + import pytest from mirakuru import SimpleExecutor, HTTPExecutor from mirakuru.compat import SIGKILL +from mirakuru.exceptions import ProcessFinishedWithError from tests import SAMPLE_DAEMON_PATH, ps_aux, TEST_SERVER_PATH @@ -38,23 +40,20 @@ def test_kill_custom_signal_kill(): def test_already_closed(): """Check that the executor cleans after itself after it exited earlier.""" - with SimpleExecutor('python3') as executor: - assert executor.running() - os.killpg(executor.process.pid, SIGKILL) - - def process_stopped(): - """Return True only only when self.process is not running.""" - return executor.running() is False - executor.wait_for(process_stopped) - assert executor.process + with pytest.raises(ProcessFinishedWithError) as excinfo: + with SimpleExecutor('python') as executor: + assert executor.running() + os.killpg(executor.process.pid, SIGKILL) + + def process_stopped(): + """Return True only only when self.process is not running.""" + return executor.running() is False + executor.wait_for(process_stopped) + assert executor.process + assert excinfo.value.exit_code == -9 assert not executor.process -@pytest.mark.xfail( - condition=sys.version_info >= (3, 8), - reason='python-daemon 2.2.3 fails with ' - '; ' - 'unxfail when a newer version is used') def test_daemons_killing(): """ Test if all subprocesses of SimpleExecutor can be killed. @@ -63,7 +62,7 @@ def test_daemons_killing(): change the process group ID. This test verifies that daemon process is killed after executor's kill(). """ - executor = SimpleExecutor(('python3', SAMPLE_DAEMON_PATH), shell=True) + executor = SimpleExecutor(('python', SAMPLE_DAEMON_PATH), shell=True) executor.start() time.sleep(2) assert executor.running() is not True, \ --- mirakuru-2.1.0.orig/tests/executors/test_tcp_executor.py +++ mirakuru-2.1.0/tests/executors/test_tcp_executor.py @@ -18,7 +18,7 @@ HTTP_SERVER = f'{HTTP_SERVER_CMD} {PORT} def test_start_and_wait(): """Test if executor await for process to accept connections.""" - command = 'bash -c "sleep 2 && nc -l -p 3000"' + command = 'bash -c "sleep 2 && nc -l 3000"' executor = TCPExecutor(command, 'localhost', port=3000, timeout=5) executor.start() @@ -32,7 +32,7 @@ def test_start_and_wait(): def test_it_raises_error_on_timeout(): """Check if TimeoutExpired gets raised correctly.""" - command = 'bash -c "sleep 10 && nc -l -p 3000"' + command = 'bash -c "sleep 10 && nc -l 3000"' executor = TCPExecutor(command, host='localhost', port=3000, timeout=2) with pytest.raises(TimeoutExpired):