diff --git a/swh/loader/core/utils.py b/swh/loader/core/utils.py --- a/swh/loader/core/utils.py +++ b/swh/loader/core/utils.py @@ -1,12 +1,18 @@ -# Copyright (C) 2018-2021 The Software Heritage developers +# Copyright (C) 2018-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import io import os import shutil +import signal +import time +import traceback +from typing import Callable +from billiard import Process, Queue # type: ignore import psutil @@ -43,3 +49,57 @@ except Exception as e: if log: log.warn("Fail to clean dangling path %s: %s", path_to_cleanup, e) + + +class CloneTimeout(Exception): + pass + + +class CloneFailure(Exception): + pass + + +def _clone_task(clone_func: Callable, errors: Queue) -> None: + try: + clone_func() + except Exception as e: + exc_buffer = io.StringIO() + traceback.print_exc(file=exc_buffer) + errors.put_nowait(exc_buffer.getvalue()) + raise e + + +def clone_with_timeout( + src: str, dest: str, clone_func: Callable, timeout: float +) -> None: + """Clone a repository with timeout. + + Args: + src: clone source + dest: clone destination + clone_func: callable that does the actual cloning + timeout: timeout in seconds + """ + errors: Queue = Queue() + process = Process(target=_clone_task, args=(clone_func, errors)) + process.start() + process.join(timeout) + + if process.is_alive(): + process.terminate() + # Give it literally a second (in successive steps of 0.1 second), + # then kill it. + # Can't use `process.join(1)` here, billiard appears to be bugged + # https://github.com/celery/billiard/issues/270 + killed = False + for _ in range(10): + time.sleep(0.1) + if not process.is_alive(): + break + else: + killed = True + os.kill(process.pid, signal.SIGKILL) + raise CloneTimeout(src, timeout, killed) + + if not errors.empty(): + raise CloneFailure(src, dest, errors.get())