diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 879c9d0..e40448e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,20 +1,20 @@ [bumpversion] commit = True tag = True message = "Release {new_version}" -current_version = 2.0.1 +current_version = 2.0.2 [bumpversion:file:setup.py] search = version='{current_version}' replace = version='{new_version}' [bumpversion:file:src/mirakuru/__init__.py] [bumpversion:file:README.rst] [bumpversion:file:CHANGES.rst] search = unreleased ---------- replace = {new_version} ---------- diff --git a/CHANGES.rst b/CHANGES.rst index 581172f..4424d4b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,172 +1,178 @@ CHANGELOG ========= +2.0.2 +---------- + +- [enhancement] Change typing definition to allow mirakuru run on + python 3.5.0 to 3.5.2 + 2.0.1 ---------- - [repackage] - mark python 3.5 as required. Should disallow installing on python 2 2.0.0 ---------- - [feature] Add UnixSocketExecutor for executors that communicate with Unix Sockets - [feature] Mirakuru is now fully type hinted - [feature] Drop support for python 2 - [feature] Allow for configuring process outputs to pipe to - [feature] OutputExecutor can now check for banner in stderr - [feature] HTTPEecutor now can check status on different method. Along with properly configured payload and headers. - [feature] Ability to set custom env vars for orchestrated process - [feature] Ability to set custom cwd path for orchestrated process - [enhancement] psutil is no longer required on cygwin 1.1.0 ---------- - [enhancement] Executor's timeout to be set for both executor's start and stop - [enhancement] It's no longer possible to hang indefinitely on the start or stop. Timeout is set to 3600 seconds by default, with values possible between `0` and `sys.maxsize` with the latter still bit longer than `2924712086` centuries. 1.0.0 ---------- - [enhancement] Do not fail if processes child throw EPERM error during clean up phase - [enhancement] Run subprocesses in shell by default on Windows - [ehnancement] Do not pass preexec_fn on windows 0.9.0 ---------- - [enhancement] Fallback to kill through SIGTERM on Windows, since SIGKILL is not available - [enhancement] detect cases where during stop process already exited, and simply clean up afterwards 0.8.3 ---------- - [enhancement] when killing the process ignore OsError with errno `no such process` as the process have already died. - [enhancement] small context manager code cleanup 0.8.2 ---------- - [bugfix] atexit cleanup_subprocesses() function now reimports needed functions 0.8.1 ---------- - [bugfix] Handle IOErrors from psutil (#112) - [bugfix] Pass global vars to atexit cleanup_subprocesses function (#111) 0.8.0 ---------- - [feature] Kill all running mirakuru subprocesses on python exit. - [enhancement] Prefer psutil library (>=4.0.0) over calling 'ps xe' command to find leaked subprocesses. 0.7.0 ---------- - [feature] HTTPExecutor enriched with the 'status' argument. It allows to define which HTTP status code(s) signify that a HTTP server is running. - [feature] Changed executor methods to return itself to allow method chaining. - [feature] Context Manager to return Executor instance, allows creating Executor instance on the fly. - [style] Migrated `%` string formating to `format()`. - [style] Explicitly numbered replacement fields in string. - [docs] Added documentation for timeouts. 0.6.1 ---------- - [refactoring] Moved source to src directory. - [fix, feature] Python 3.5 fixes. - [fix] Docstring changes for updated pep257. 0.6.0 ---------- - [fix] Modify MANIFEST to prune tests folder. - [feature] HTTPExecutor will now set the default 80 if not present in a URL. - [feature] Detect subprocesses exiting erroneously while polling the checks and error early. - [fix] Make test_forgotten_stop pass by preventing the shell from optimizing forking out. 0.5.0 ---------- - [style] Corrected code to conform with W503, D210 and E402 linters errors as reported by pylama `6.3.1`. - [feature] Introduced a hack that kills all subprocesses of executor process. It requires 'ps xe -ww' command being available in OS otherwise logs error. - [refactoring] Classes name convention change. Executor class got renamed into SimpleExecutor and StartCheckExecutor class got renamed into Executor. 0.4.0 ------- - [feature] Ability to set up custom signal for stopping and killing processes managed by executors. - [feature] Replaced explicit parameters with keywords for kwargs handled by basic Executor init method. - [feature] Executor now accepts both list and string as a command. - [fix] Even it's not recommended to import all but `from mirakuru import *` didn't worked. Now it's fixed. - [tests] increased tests coverage. Even test cover 100% of code it doesn't mean they cover 100% of use cases! - [code quality] Increased Pylint code evaluation. 0.3.0 ------- - [feature] Introduced PidExecutor that waits for specified file to be created. - [feature] Provided PyPy compatibility. - [fix] Closing all resources explicitly. 0.2.0 ------- - [fix] Kill all children processes of Executor started with shell=True. - [feature] Executors are now context managers - to start executors for given context. - [feature] Executor.stopped - context manager for stopping executors for given context. - [feature] HTTPExecutor and TCPExecutor before .start() check whether port is already used by other processes and raise AlreadyRunning if detects it. - [refactoring] Moved python version conditional imports into compat.py module. 0.1.4 ------- - [fix] Fixed an issue where setting shell to True would execute only part of the command. 0.1.3 ------- - [fix] Fixed an issue where OutputExecutor would hang, if started process stopped producing output. 0.1.2 ------- - [fix] Removed leftover sleep from TCPExecutor._wait_for_connection. 0.1.1 ------- - [fix] Fixed `MANIFEST.in`. - Updated packaging options. 0.1.0 ------- - Exposed process attribute on Executor. - Exposed port and host on TCPExecutor. - Exposed URL on HTTPExecutor. - Simplified package structure. - Simplified executors operating API. - Updated documentation. - Added docblocks for every function. - Applied license headers. - Stripped orchestrators. - Forked off from `summon_process`. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..378f68d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.5.2 + +WORKDIR /usr/src/app + +RUN pip install --upgrade pip +COPY . . +RUN pip install --no-cache-dir -r requirements-lint.txt +RUN pip install --no-cache-dir -r requirements-test.txt +RUN pip install --no-cache-dir -e .[tests] + +COPY . . + +CMD [ "pytest" ] diff --git a/README.rst b/README.rst index 98a9a84..36a50db 100644 --- a/README.rst +++ b/README.rst @@ -1,123 +1,123 @@ mirakuru ======== Mirakuru is a process orchestration tool designed for functional and integration tests. Maybe you want to be able to start a database before you start your program or maybe you just need to set additional services up for your tests. This is where you should consider using **mirakuru** to add superpowers to your program or tests. .. image:: https://img.shields.io/pypi/v/mirakuru.svg :target: https://pypi.python.org/pypi/mirakuru/ :alt: Latest PyPI version -.. image:: https://readthedocs.org/projects/mirakuru/badge/?version=v2.0.1 - :target: http://mirakuru.readthedocs.io/en/v2.0.1/ +.. image:: https://readthedocs.org/projects/mirakuru/badge/?version=v2.0.2 + :target: http://mirakuru.readthedocs.io/en/v2.0.2/ :alt: Documentation Status .. image:: https://img.shields.io/pypi/wheel/mirakuru.svg :target: https://pypi.python.org/pypi/mirakuru/ :alt: Wheel Status .. image:: https://img.shields.io/pypi/pyversions/mirakuru.svg :target: https://pypi.python.org/pypi/mirakuru/ :alt: Supported Python Versions .. image:: https://img.shields.io/pypi/l/mirakuru.svg :target: https://pypi.python.org/pypi/mirakuru/ :alt: License Package status -------------- -.. image:: https://travis-ci.org/ClearcodeHQ/mirakuru.svg?branch=v2.0.1 +.. image:: https://travis-ci.org/ClearcodeHQ/mirakuru.svg?branch=v2.0.2 :target: https://travis-ci.org/ClearcodeHQ/mirakuru :alt: Tests -.. image:: https://coveralls.io/repos/ClearcodeHQ/mirakuru/badge.png?branch=v2.0.1 - :target: https://coveralls.io/r/ClearcodeHQ/mirakuru?branch=v2.0.1 +.. image:: https://coveralls.io/repos/ClearcodeHQ/mirakuru/badge.png?branch=v2.0.2 + :target: https://coveralls.io/r/ClearcodeHQ/mirakuru?branch=v2.0.2 :alt: Coverage Status -.. image:: https://requires.io/github/ClearcodeHQ/mirakuru/requirements.svg?tag=v2.0.1 - :target: https://requires.io/github/ClearcodeHQ/mirakuru/requirements/?tag=v2.0.1 +.. image:: https://requires.io/github/ClearcodeHQ/mirakuru/requirements.svg?tag=v2.0.2 + :target: https://requires.io/github/ClearcodeHQ/mirakuru/requirements/?tag=v2.0.2 :alt: Requirements Status About ----- In a project that relies on multiple processes there might be a need to guard code with tests that verify interprocess communication. So one needs to set up all of required databases, auxiliary and application services to verify their cooperation. Synchronising (or orchestrating) test procedure with tested processes might be a hell. If so, then **mirakuru** is what you need. ``Mirakuru`` starts your process and waits for the clear indication that it's running. Library provides seven executors to fit different cases: * **SimpleExecutor** - starts a process and does not wait for anything. It is useful to stop or kill a process and its subprocesses. Base class for all the rest of executors. * **Executor** - base class for executors verifying if a process has started. * **OutputExecutor** - waits for a specified output to be printed by a process. * **TCPExecutor** - waits for the ability to connect through TCP with a process. * **UnixSocketExecutor** - waits for the ability to connect through Unix socket with a process * **HTTPExecutor** - waits for a successful HEAD request (and TCP before). * **PidExecutor** - waits for a specified .pid file to exist. .. code-block:: python from mirakuru import HTTPExecutor from httplib import HTTPConnection, OK def test_it_works(): # The ``./http_server`` here launches some HTTP server on the 6543 port, # but naturally it is not immediate and takes a non-deterministic time: executor = HTTPExecutor("./http_server", url="http://127.0.0.1:6543/") # Start the server and wait for it to run (blocking): executor.start() # Here the server should be running! conn = HTTPConnection("127.0.0.1", 6543) conn.request("GET", "/") assert conn.getresponse().status is OK executor.stop() A command by which executor spawns a process can be defined by either string or list. .. code-block:: python # command as string TCPExecutor('python -m smtpd -n -c DebuggingServer localhost:1025', host='localhost', port=1025) # command as list TCPExecutor( ['python', '-m', 'smtpd', '-n', '-c', 'DebuggingServer', 'localhost:1025'], host='localhost', port=1025 ) Authors ------- The project was firstly developed by `Mateusz Lenik `_ as the `summon_process `_. Later forked, renamed into **mirakuru** and tended to by The A Room @ `Clearcode `_ and `the other authors `_. License ------- ``mirakuru`` is licensed under LGPL license, version 3. Contributing and reporting bugs ------------------------------- Source code is available at: `ClearcodeHQ/mirakuru `_. Issue tracker is located at `GitHub Issues `_. Projects `PyPI page `_. When contributing, don't forget to add your name to the AUTHORS.rst file. diff --git a/setup.py b/setup.py index 9855ceb..b07a79f 100644 --- a/setup.py +++ b/setup.py @@ -1,93 +1,93 @@ # Copyright (C) 2014 by Clearcode # and associates (see AUTHORS). # This file is part of mirakuru. # mirakuru is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # mirakuru is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # You should have received a copy of the GNU Lesser General Public License # along with mirakuru. If not, see . """Mirakuru installation module.""" import os from setuptools import setup, find_packages here = os.path.dirname(__file__) requirements = [ # psutil is used to find processes leaked during termination. # It runs on many platforms but not Cygwin: # . 'psutil>=4.0.0; sys_platform != "cygwin"', ] tests_require = ( 'pytest==4.6.3', # tests framework used 'pytest-cov==2.7.1', # coverage reports to verify tests quality 'mock==3.0.5', # tests mocking tool 'python-daemon==2.2.3', # used in test for easy creation of daemons ) extras_require = { 'docs': ['sphinx'], 'tests': tests_require, } def read(fname): """ Read filename. :param str fname: name of a file to read """ return open(os.path.join(here, fname)).read() setup( name='mirakuru', - version='2.0.1', + version='2.0.2', description='Process executor for tests.', long_description=( read('README.rst') + '\n\n' + read('CHANGES.rst') ), keywords='process executor tests summon_process', url='https://github.com/ClearcodeHQ/mirakuru', author='Clearcode - The A Room', author_email='thearoom@clearcode.cc', license='LGPL', python_requires='>=3.5', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Web Environment', 'Intended Audience :: Developers', 'License :: OSI Approved :: ' 'GNU Lesser General Public License v3 or later (LGPLv3+)', 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Software Development :: Testing', ], package_dir={'': 'src'}, packages=find_packages('src'), install_requires=requirements, tests_require=tests_require, test_suite='tests', include_package_data=True, zip_safe=False, extras_require=extras_require, ) diff --git a/src/mirakuru/__init__.py b/src/mirakuru/__init__.py index 0b0a573..1c9160e 100644 --- a/src/mirakuru/__init__.py +++ b/src/mirakuru/__init__.py @@ -1,53 +1,53 @@ # Copyright (C) 2014 by Clearcode # and associates (see AUTHORS). # This file is part of mirakuru. # mirakuru is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # mirakuru is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # You should have received a copy of the GNU Lesser General Public License # along with mirakuru. If not, see . """Mirakuru main module.""" import logging from mirakuru.base import Executor, SimpleExecutor from mirakuru.output import OutputExecutor from mirakuru.tcp import TCPExecutor from mirakuru.http import HTTPExecutor from mirakuru.pid import PidExecutor from mirakuru.exceptions import ( ExecutorError, TimeoutExpired, AlreadyRunning, ProcessExitedWithError, ) -__version__ = '2.0.1' +__version__ = '2.0.2' __all__ = ( 'Executor', 'SimpleExecutor', 'OutputExecutor', 'TCPExecutor', 'HTTPExecutor', 'PidExecutor', 'ExecutorError', 'TimeoutExpired', 'AlreadyRunning', 'ProcessExitedWithError', ) # Set default logging handler to avoid "No handler found" warnings. logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/src/mirakuru/base.py b/src/mirakuru/base.py index b5e0ea6..c3ae54f 100644 --- a/src/mirakuru/base.py +++ b/src/mirakuru/base.py @@ -1,526 +1,526 @@ # Copyright (C) 2014 by Clearcode # and associates (see AUTHORS). # This file is part of mirakuru. # mirakuru is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # mirakuru is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # You should have received a copy of the GNU Lesser General Public License # along with mirakuru. If not, see . """Executor with the core functionality.""" import atexit from contextlib import contextmanager import logging import os import shlex import signal import subprocess import time import uuid import errno import platform from types import TracebackType from typing import ( Union, IO, Any, List, Tuple, Optional, Dict, TypeVar, Type, Set, Iterator, Callable ) from mirakuru.base_env import processes_with_env from mirakuru.exceptions import ( AlreadyRunning, ProcessExitedWithError, TimeoutExpired, ) from mirakuru.compat import SIGKILL log = logging.getLogger(__name__) # pylint: disable=invalid-name ENV_UUID = 'mirakuru_uuid' """ Name of the environment variable used by mirakuru to mark its subprocesses. """ IGNORED_ERROR_CODES = [errno.ESRCH] if platform.system() == 'Darwin': IGNORED_ERROR_CODES = [errno.ESRCH, errno.EPERM] # Type variables used for self in functions returning self, so it's correctly # typed in derived classes. SimpleExecutorType = TypeVar("SimpleExecutorType", bound="SimpleExecutor") ExecutorType = TypeVar("ExecutorType", bound="Executor") @atexit.register def cleanup_subprocesses() -> None: """On python exit: find possibly running subprocesses and kill them.""" # pylint: disable=redefined-outer-name, reimported # atexit functions tends to loose global imports sometimes so reimport # everything what is needed again here: import os import errno from mirakuru.base_env import processes_with_env from mirakuru.compat import SIGKILL pids = processes_with_env(ENV_UUID, str(os.getpid())) for pid in pids: try: os.kill(pid, SIGKILL) except OSError as err: if err.errno != errno.ESRCH: print("Can not kill the", pid, "leaked process", err) class SimpleExecutor: # pylint:disable=too-many-instance-attributes """Simple subprocess executor with start/stop/kill functionality.""" def __init__( # pylint:disable=too-many-arguments self, command: Union[str, List[str], Tuple[str, ...]], cwd: Optional[str] = None, shell: bool = False, timeout: Union[int, float] = 3600, sleep: float = 0.1, sig_stop: int = signal.SIGTERM, sig_kill: int = SIGKILL, envvars: Optional[Dict[str, str]] = None, stdin: Union[None, int, IO[Any]] = subprocess.PIPE, stdout: Union[None, int, IO[Any]] = subprocess.PIPE, stderr: Union[None, int, IO[Any]] = None ) -> None: """ Initialize executor. :param (str, list) command: command to be run by the subprocess :param str cwd: current working directory to be set for executor :param bool shell: same as the `subprocess.Popen` shell definition. On Windows always set to True. :param int timeout: number of seconds to wait for the process to start or stop. :param float sleep: how often to check for start/stop condition :param int sig_stop: signal used to stop process run by the executor. default is `signal.SIGTERM` :param int sig_kill: signal used to kill process run by the executor. default is `signal.SIGKILL` (`signal.SIGTERM` on Windows) :param dict envvars: Additional environment variables :param int stdin: file descriptor for stdin :param int stdout: file descriptor for stdout :param int stderr: file descriptor for stderr .. note:: **timeout** set for an executor is valid for all the level of waits on the way up. That means that if some more advanced executor establishes the timeout to 10 seconds and it will take 5 seconds for the first check, second check will only have 5 seconds left. Your executor will raise an exception if something goes wrong during this time. The default value of timeout is ``None``, so it is a good practice to set this. """ if isinstance(command, (list, tuple)): self.command = ' '.join(command) """Command that the executor runs.""" self.command_parts = command else: self.command = command self.command_parts = shlex.split(command) self._cwd = cwd self._shell = True if platform.system() != 'Windows': self._shell = shell self._timeout = timeout self._sleep = sleep self._sig_stop = sig_stop self._sig_kill = sig_kill self._envvars = envvars or {} self._stdin = stdin self._stdout = stdout self._stderr = stderr self._endtime = None # type: Optional[float] self.process = None # type: Optional[subprocess.Popen] """A :class:`subprocess.Popen` instance once process is started.""" self._uuid = '{0}:{1}'.format(os.getpid(), uuid.uuid4()) def __enter__(self: SimpleExecutorType) -> SimpleExecutorType: """ Enter context manager starting the subprocess. :returns: itself :rtype: SimpleExecutor """ return self.start() def __exit__(self, - exc_type: Optional[Type[BaseException]], + exc_type: Type[BaseException], exc_value: Optional[BaseException], traceback: Optional[TracebackType]) -> None: """Exit context manager stopping the subprocess.""" self.stop() def running(self) -> bool: """ Check if executor is running. :returns: SimpleExecutorTyperue if process is running, False otherwise :rtype: bool """ if self.process is None: return False return self.process.poll() is None @property def _popen_kwargs(self) -> Dict[str, Any]: """ Get kwargs for the process instance. .. note:: We want to open ``stdin``, ``stdout`` and ``stderr`` as text streams in universal newlines mode, so we have to set ``universal_newlines`` to ``True``. :return: """ kwargs = {} # type: Dict[str, Any] if self._stdin: kwargs['stdin'] = self._stdin if self._stdout: kwargs['stdout'] = self._stdout if self._stderr: kwargs['stderr'] = self._stderr kwargs['universal_newlines'] = True kwargs['shell'] = self._shell env = os.environ.copy() env.update(self._envvars) # Trick with marking subprocesses with an environment variable. # # There is no easy way to recognize all subprocesses that were # spawned during lifetime of a certain subprocess so mirakuru does # this hack in order to mark who was the original parent. Even if # some subprocess got daemonized or changed original process group # mirakuru will be able to find it by this environment variable. # # There may be a situation when some subprocess will abandon # original envs from parents and then it won't be later found. env[ENV_UUID] = self._uuid kwargs['env'] = env kwargs['cwd'] = self._cwd if platform.system() != 'Windows': kwargs['preexec_fn'] = os.setsid return kwargs def start(self: SimpleExecutorType) -> SimpleExecutorType: """ Start defined process. After process gets started, timeout countdown begins as well. :returns: itself :rtype: SimpleExecutor """ if self.process is None: command = \ self.command # type: Union[str, List[str], Tuple[str, ...]] if not self._shell: command = self.command_parts self.process = subprocess.Popen( command, **self._popen_kwargs ) self._set_timeout() return self def _set_timeout(self) -> None: """Set timeout for possible wait.""" self._endtime = time.time() + self._timeout def _clear_process(self) -> None: """ Close stdin/stdout of subprocess. It is required because of ResourceWarning in Python 3. """ if self.process: self.process.__exit__(None, None, None) self.process = None self._endtime = None def _kill_all_kids(self, sig: int) -> Set[int]: """ Kill all subprocesses (and its subprocesses) that executor started. This function tries to kill all leftovers in process tree that current executor may have left. It uses environment variable to recognise if process have origin in this Executor so it does not give 100 % and some daemons fired by subprocess may still be running. :param int sig: signal used to stop process run by executor. :return: process ids (pids) of killed processes :rtype: set """ pids = processes_with_env(ENV_UUID, self._uuid) for pid in pids: log.debug("Killing process %d ...", pid) try: os.kill(pid, sig) except OSError as err: if err.errno in IGNORED_ERROR_CODES: # the process has died before we tried to kill it. pass else: raise log.debug("Killed process %d.", pid) return pids def stop(self: SimpleExecutorType, sig: int = None) -> SimpleExecutorType: """ Stop process running. Wait 10 seconds for the process to end, then just kill it. :param int sig: signal used to stop process run by executor. None for default. :returns: itself :rtype: SimpleExecutor .. note:: When gathering coverage for the subprocess in tests, you have to allow subprocesses to end gracefully. """ if self.process is None: return self if sig is None: sig = self._sig_stop try: os.killpg(self.process.pid, sig) except OSError as err: if err.errno in IGNORED_ERROR_CODES: pass else: raise def process_stopped() -> bool: """Return True only only when self.process is not running.""" return self.running() is False self._set_timeout() try: self.wait_for(process_stopped) except TimeoutExpired: # at this moment, process got killed, pass self._kill_all_kids(sig) self._clear_process() return self @contextmanager def stopped(self: SimpleExecutorType) -> Iterator[SimpleExecutorType]: """ Stop process for given context and starts it afterwards. Allows for easier writing resistance integration tests whenever one of the service fails. :yields: itself :rtype: SimpleExecutor """ if self.running(): self.stop() yield self self.start() def kill( self: SimpleExecutorType, wait: bool = True, sig: Optional[int] = None) -> SimpleExecutorType: """ Kill the process if running. :param bool wait: set to `True` to wait for the process to end, or False, to simply proceed after sending signal. :param int sig: signal used to kill process run by the executor. None by default. :returns: itself :rtype: SimpleExecutor """ if sig is None: sig = self._sig_kill if self.process and self.running(): os.killpg(self.process.pid, sig) if wait: self.process.wait() self._kill_all_kids(sig) self._clear_process() return self def output(self) -> Optional[IO[Any]]: """Return subprocess output.""" if self.process is not None: return self.process.stdout return None # pragma: no cover def err_output(self) -> Optional[IO[Any]]: """Return subprocess stderr.""" if self.process is not None: return self.process.stderr return None # pragma: no cover def wait_for( self: SimpleExecutorType, wait_for: Callable[[], bool]) -> SimpleExecutorType: """ Wait for callback to return True. Simply returns if wait_for condition has been met, raises TimeoutExpired otherwise and kills the process. :param callback wait_for: callback to call :raises: mirakuru.exceptions.TimeoutExpired :returns: itself :rtype: SimpleExecutor """ while self.check_timeout(): if wait_for(): return self time.sleep(self._sleep) self.kill() raise TimeoutExpired(self, timeout=self._timeout) def check_timeout(self) -> bool: """ Check if timeout has expired. Returns True if there is no timeout set or the timeout has not expired. Kills the process and raises TimeoutExpired exception otherwise. This method should be used in while loops waiting for some data. :return: True if timeout expired, False if not :rtype: bool """ return self._endtime is None or time.time() <= self._endtime def __del__(self) -> None: """Cleanup subprocesses created during Executor lifetime.""" try: if self.process: self.kill() except Exception: # pragma: no cover print("*" * 80) print("Exception while deleting Executor. '" "It is strongly suggested that you use") print("it as a context manager instead.") print("*" * 80) raise def __repr__(self) -> str: """Return unambiguous executor representation.""" command = self.command if len(command) > 10: command = command[:10] + '...' return '<{module}.{executor}: "{command}" {id}>'.format( module=self.__class__.__module__, executor=self.__class__.__name__, command=command, id=hex(id(self)) ) def __str__(self) -> str: """Return readable executor representation.""" return '<{module}.{executor}: "{command}">'.format( module=self.__class__.__module__, executor=self.__class__.__name__, command=self.command ) class Executor(SimpleExecutor): """Base class for executors with a pre- and after-start checks.""" def pre_start_check(self) -> bool: """ Check process before the start of executor. Should be overridden in order to return True when some other executor (or process) has already started with the same configuration. :rtype: bool """ raise NotImplementedError def start(self: ExecutorType) -> ExecutorType: """ Start executor with additional checks. Checks if previous executor isn't running then start process (executor) and wait until it's started. :returns: itself :rtype: Executor """ if self.pre_start_check(): # Some other executor (or process) is running with same config: raise AlreadyRunning(self) super(Executor, self).start() self.wait_for(self.check_subprocess) return self def check_subprocess(self) -> bool: """ Make sure the process didn't exit with an error and run the checks. :rtype: bool :return: the actual check status or False before starting the process :raise ProcessExitedWithError: when the main process exits with an error """ if self.process is None: # pragma: no cover # No process was started. return False exit_code = self.process.poll() if exit_code is not None and exit_code != 0: # The main process exited with an error. Clean up the children # if any. self._kill_all_kids(self._sig_kill) self._clear_process() raise ProcessExitedWithError(self, exit_code) return self.after_start_check() def after_start_check(self) -> bool: """ Check process after the start of executor. Should be overridden in order to return boolean value if executor can be treated as started. :rtype: bool """ raise NotImplementedError