diff --git a/PKG-INFO b/PKG-INFO index 75dba8a..0fc831a 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,25 +1,29 @@ Metadata-Version: 2.1 Name: swh.icinga_plugins -Version: 0.3.0 +Version: 0.3.1 Summary: Icinga plugins for Software Heritage infrastructure monitoring Home-page: https://forge.softwareheritage.org/diffusion/swh-icinga-plugins Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-icinga-plugins -Description: swh-icinga-plugins - ================== - - Scripts for end-to-end monitoring of the SWH infrastructure - Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 3 - Alpha Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing +License-File: LICENSE +License-File: AUTHORS + +swh-icinga-plugins +================== + +Scripts for end-to-end monitoring of the SWH infrastructure + + diff --git a/requirements.txt b/requirements.txt index 7748e9d..fdd5330 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ +click psycopg2 requests +types-click +types-requests diff --git a/swh.icinga_plugins.egg-info/PKG-INFO b/swh.icinga_plugins.egg-info/PKG-INFO index c5f4061..c2d0067 100644 --- a/swh.icinga_plugins.egg-info/PKG-INFO +++ b/swh.icinga_plugins.egg-info/PKG-INFO @@ -1,25 +1,29 @@ Metadata-Version: 2.1 Name: swh.icinga-plugins -Version: 0.3.0 +Version: 0.3.1 Summary: Icinga plugins for Software Heritage infrastructure monitoring Home-page: https://forge.softwareheritage.org/diffusion/swh-icinga-plugins Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-icinga-plugins -Description: swh-icinga-plugins - ================== - - Scripts for end-to-end monitoring of the SWH infrastructure - Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 3 - Alpha Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing +License-File: LICENSE +License-File: AUTHORS + +swh-icinga-plugins +================== + +Scripts for end-to-end monitoring of the SWH infrastructure + + diff --git a/swh.icinga_plugins.egg-info/requires.txt b/swh.icinga_plugins.egg-info/requires.txt index ac6f964..f585ba2 100644 --- a/swh.icinga_plugins.egg-info/requires.txt +++ b/swh.icinga_plugins.egg-info/requires.txt @@ -1,10 +1,13 @@ +click psycopg2 requests +types-click +types-requests swh.core[http]>=0.3 swh.deposit>=0.3 swh.storage>=0.0.162 [testing] pytest pytest-mock requests-mock diff --git a/swh/__init__.py b/swh/__init__.py index f14e196..8d9f151 100644 --- a/swh/__init__.py +++ b/swh/__init__.py @@ -1,4 +1,4 @@ from pkgutil import extend_path -from typing import Iterable +from typing import List -__path__ = extend_path(__path__, __name__) # type: Iterable[str] +__path__: List[str] = extend_path(__path__, __name__) diff --git a/swh/icinga_plugins/save_code_now.py b/swh/icinga_plugins/save_code_now.py index 02ed4fa..131c080 100644 --- a/swh/icinga_plugins/save_code_now.py +++ b/swh/icinga_plugins/save_code_now.py @@ -1,113 +1,113 @@ # Copyright (C) 2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import time from typing import Dict, List import requests from .base_check import BaseCheck REPORT_MSG = "Save code now request for origin" WAITING_STATUSES = ("not yet scheduled", "running", "scheduled") class SaveCodeNowCheck(BaseCheck): TYPE = "SAVECODENOW" DEFAULT_WARNING_THRESHOLD = 60 DEFAULT_CRITICAL_THRESHOLD = 120 def __init__(self, obj: Dict, origin: str, visit_type: str) -> None: super().__init__(obj) self.api_url = obj["swh_web_url"].rstrip("/") self.poll_interval = obj["poll_interval"] self.origin = origin self.visit_type = visit_type @staticmethod def api_url_scn(root_api_url: str, origin: str, visit_type: str) -> str: """Compute the save code now api url for a given origin""" return f"{root_api_url}/api/1/origin/save/{visit_type}/url/{origin}/" def main(self) -> int: """Scenario description: 1. Requests a save code now request via the api for origin self.origin with type self.visit_type. 2. Polling regularly at self.poll_interval seconds the completion status. 3. When either succeeded, failed or threshold exceeded, report approximate time of completion. This will warn if thresholds are exceeded. """ start_time: float = time.time() total_time: float = 0.0 scn_url = self.api_url_scn(self.api_url, self.origin, self.visit_type) response = requests.post(scn_url) assert response.status_code == 200, (response, response.text) result: Dict = response.json() status_key = "save_task_status" request_date = result["save_request_date"] origin_info = (self.visit_type, self.origin) while result[status_key] in WAITING_STATUSES: time.sleep(self.poll_interval) response = requests.get(scn_url) assert ( response.status_code == 200 - ), "Unexpected response: {response}, {response.text}" + ), f"Unexpected response: {response}, {response.text}" raw_result: List[Dict] = response.json() assert len(raw_result) > 0, f"Unexpected result: {raw_result}" if len(raw_result) > 1: # retrieve only the one status result we are interested in result = next( filter(lambda r: r["save_request_date"] == request_date, raw_result) ) else: result = raw_result[0] # this because the api can return multiple entries for the same origin assert result["save_request_date"] == request_date total_time = time.time() - start_time if total_time > self.critical_threshold: self.print_result( "CRITICAL", f"{REPORT_MSG} {origin_info} took more than {total_time:.2f}s " f'and has status: {result["save_task_status"]}.', total_time=total_time, ) return 2 if result[status_key] == "succeeded": (status_code, status) = self.get_status(total_time) self.print_result( status, f"{REPORT_MSG} {origin_info} took {total_time:.2f}s and succeeded.", total_time=total_time, ) return status_code elif result[status_key] == "failed": self.print_result( "CRITICAL", f"{REPORT_MSG} {origin_info} took {total_time:.2f}s and failed.", total_time=total_time, ) return 2 else: self.print_result( "CRITICAL", f"{REPORT_MSG} {origin_info} took {total_time:.2f}s " "and resulted in unsupported status: " f"{result['save_request_status']} ; {result[status_key]}.", total_time=total_time, ) return 2 diff --git a/swh/icinga_plugins/tests/web_scenario.py b/swh/icinga_plugins/tests/web_scenario.py index 26c8d81..e4225cb 100644 --- a/swh/icinga_plugins/tests/web_scenario.py +++ b/swh/icinga_plugins/tests/web_scenario.py @@ -1,90 +1,92 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Wrapper around requests-mock to mock successive responses from a web service. Tests can build successive steps by calling :py:meth:`WebScenario.add_step` with specifications of what endpoints should be called and in what order.""" from dataclasses import dataclass import json from typing import Callable, List, Optional, Set import requests_mock @dataclass(frozen=True) class Step: expected_method: str expected_url: str response: object status_code: int = 200 callback: Optional[Callable[[], int]] = None @dataclass(frozen=True) class Endpoint: method: str url: str class WebScenario: """Stores the state of the successive calls to the web service expected by tests.""" _steps: List[Step] _endpoints: Set[Endpoint] _current_step: int def __init__(self): self._steps = [] self._endpoints = set() self._current_step = 0 def add_endpoint(self, *args, **kwargs): """Adds an endpoint to be mocked. Arguments are the same as :py:class:Endpoint. """ self._endpoints.add(Endpoint(*args, **kwargs)) def add_step(self, *args, **kwargs): """Adds an expected call to the list of expected calls. Also automatically calls :py:meth:`add_endpoint` so the associated endpoint is mocked. Arguments are the same as :py:class:`Step`. """ step = Step(*args, **kwargs) self._steps.append(step) self.add_endpoint(step.expected_method, step.expected_url) def install_mock(self, mocker: requests_mock.Mocker): """Mocks entrypoints registered with :py:meth:`add_endpoint` (or :py:meth:`add_step`) using the provided mocker. """ for endpoint in self._endpoints: mocker.register_uri( - endpoint.method.upper(), endpoint.url, text=self._request_callback + endpoint.method.upper(), + endpoint.url, + text=self._request_callback, # type: ignore # stubs are too strict ) def _request_callback(self, request, context): step = self._steps[self._current_step] assert request.url == step.expected_url assert request.method.upper() == step.expected_method.upper() self._current_step += 1 context.status_code = step.status_code if step.callback: step.callback() if isinstance(step.response, str): return step.response else: return json.dumps(step.response) diff --git a/tox.ini b/tox.ini index 69383cb..f9c01f6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,34 +1,73 @@ [tox] envlist=black,flake8,mypy,py3 [testenv:py3] deps = .[testing] pytest-cov commands = pytest --doctest-modules \ {envsitepackagesdir}/swh/icinga_plugins \ --cov={envsitepackagesdir}/swh/icinga_plugins \ --cov-branch {posargs} [testenv:black] skip_install = true deps = black==19.10b0 commands = {envpython} -m black --check swh [testenv:flake8] skip_install = true deps = flake8 commands = {envpython} -m flake8 [testenv:mypy] skip_install = true deps = .[testing] mypy commands = mypy swh + +# build documentation outside swh-environment using the current +# git HEAD of swh-docs, is executed on CI for each diff to prevent +# breaking doc build +[testenv:sphinx] +whitelist_externals = make +usedevelop = true +extras = + testing +deps = + # fetch and install swh-docs in develop mode + -e git+https://forge.softwareheritage.org/source/swh-docs#egg=swh.docs + +setenv = + SWH_PACKAGE_DOC_TOX_BUILD = 1 + # turn warnings into errors + SPHINXOPTS = -W +commands = + make -I ../.tox/sphinx/src/swh-docs/swh/ -C docs + + +# build documentation only inside swh-environment using local state +# of swh-docs package +[testenv:sphinx-dev] +whitelist_externals = make +usedevelop = true +extras = + testing +deps = + # install swh-docs in develop mode + -e ../swh-docs + +setenv = + SWH_PACKAGE_DOC_TOX_BUILD = 1 + # turn warnings into errors + SPHINXOPTS = -W +commands = + make -I ../.tox/sphinx-dev/src/swh-docs/swh/ -C docs +