diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,25 +31,7 @@ hooks: - id: isort -# unfortunately, we are far from being able to enable this... -# - repo: https://github.com/PyCQA/pydocstyle.git -# rev: 4.0.0 -# hooks: -# - id: pydocstyle -# name: pydocstyle -# description: pydocstyle is a static analysis tool for checking compliance with Python docstring conventions. -# entry: pydocstyle --convention=google -# language: python -# types: [python] - -# black requires py3.6+ -#- repo: https://github.com/python/black -# rev: 19.3b0 -# hooks: -# - id: black -# language_version: python3 -#- repo: https://github.com/asottile/blacken-docs -# rev: v1.0.0-1 -# hooks: -# - id: blacken-docs -# additional_dependencies: [black==19.3b0] +- repo: https://github.com/python/black + rev: 19.10b0 + hooks: + - id: black diff --git a/pyproject.toml b/pyproject.toml --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,6 @@ +[tool.black] +target-version = ['py37'] + [tool.isort] multi_line_output = 3 include_trailing_comma = true diff --git a/setup.cfg b/setup.cfg new file mode 100644 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[flake8] +# E203: whitespaces before ':' +# E231: missing whitespace after ',' +# W503: line break before binary operator +ignore = E203,E231,W503 +max-line-length = 88 diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -12,15 +12,15 @@ here = path.abspath(path.dirname(__file__)) # Get the long description from the README file -with open(path.join(here, 'README.md'), encoding='utf-8') as f: +with open(path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: - reqf = 'requirements-%s.txt' % name + reqf = "requirements-%s.txt" % name else: - reqf = 'requirements.txt' + reqf = "requirements.txt" requirements = [] if not path.exists(reqf): @@ -29,7 +29,7 @@ with open(reqf) as f: for line in f.readlines(): line = line.strip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue requirements.append(line) return requirements @@ -37,33 +37,31 @@ # package generated static assets as module data files data_files = [] -for root, _, files in walk('data/'): +for root, _, files in walk("data/"): root_files = [path.join(root, i) for i in files] - data_files.append((path.join('share/swh/icinga-plugins', root), - root_files)) + data_files.append((path.join("share/swh/icinga-plugins", root), root_files)) setup( - name='swh.icinga_plugins', - description='Icinga plugins for Software Heritage infrastructure ' - 'monitoring', + name="swh.icinga_plugins", + description="Icinga plugins for Software Heritage infrastructure " "monitoring", long_description=long_description, - long_description_content_type='text/markdown', + long_description_content_type="text/markdown", python_requires=">=3.7", - author='Software Heritage developers', - author_email='swh-devel@inria.fr', - url='https://forge.softwareheritage.org/diffusion/swh-icinga-plugins', + author="Software Heritage developers", + author_email="swh-devel@inria.fr", + url="https://forge.softwareheritage.org/diffusion/swh-icinga-plugins", packages=find_packages(), # packages's modules - install_requires=parse_requirements() + parse_requirements('swh'), - tests_require=parse_requirements('test'), - setup_requires=['setuptools-scm'], + install_requires=parse_requirements() + parse_requirements("swh"), + tests_require=parse_requirements("test"), + setup_requires=["setuptools-scm"], use_scm_version=True, - extras_require={'testing': parse_requirements('test')}, + extras_require={"testing": parse_requirements("test")}, include_package_data=True, - entry_points=''' + entry_points=""" [swh.cli.subcommands] icinga_plugins=swh.icinga_plugins.cli:cli - ''', + """, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", @@ -72,10 +70,9 @@ "Development Status :: 3 - Alpha", ], project_urls={ - 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', - 'Funding': 'https://www.softwareheritage.org/donate', - 'Source': - 'https://forge.softwareheritage.org/source/swh-icinga-plugins', + "Bug Reports": "https://forge.softwareheritage.org/maniphest", + "Funding": "https://www.softwareheritage.org/donate", + "Source": "https://forge.softwareheritage.org/source/swh-icinga-plugins", }, - data_files=data_files + data_files=data_files, ) diff --git a/swh/icinga_plugins/base_check.py b/swh/icinga_plugins/base_check.py --- a/swh/icinga_plugins/base_check.py +++ b/swh/icinga_plugins/base_check.py @@ -7,19 +7,21 @@ class BaseCheck: def __init__(self, obj): self.warning_threshold = obj.get( - 'warning_threshold', self.DEFAULT_WARNING_THRESHOLD) + "warning_threshold", self.DEFAULT_WARNING_THRESHOLD + ) self.critical_threshold = obj.get( - 'critical_threshold', self.DEFAULT_CRITICAL_THRESHOLD) + "critical_threshold", self.DEFAULT_CRITICAL_THRESHOLD + ) def get_status(self, value): if self.critical_threshold and value >= self.critical_threshold: - return (2, 'CRITICAL') + return (2, "CRITICAL") elif self.warning_threshold and value >= self.warning_threshold: - return (1, 'WARNING') + return (1, "WARNING") else: - return (0, 'OK') + return (0, "OK") def print_result(self, status_type, status_string, **metrics): - print(f'{self.TYPE} {status_type} - {status_string}') + print(f"{self.TYPE} {status_type} - {status_string}") for (metric_name, metric_value) in sorted(metrics.items()): print(f"| '{metric_name}' = {metric_value:.2f}s") diff --git a/swh/icinga_plugins/cli.py b/swh/icinga_plugins/cli.py --- a/swh/icinga_plugins/cli.py +++ b/swh/icinga_plugins/cli.py @@ -12,67 +12,88 @@ from swh.core.cli import CONTEXT_SETTINGS -@click.group(name='icinga_plugins', context_settings=CONTEXT_SETTINGS) -@click.option('-w', '--warning', type=int, - help='Warning threshold.') -@click.option('-c', '--critical', type=int, - help='Critical threshold.') +@click.group(name="icinga_plugins", context_settings=CONTEXT_SETTINGS) +@click.option("-w", "--warning", type=int, help="Warning threshold.") +@click.option("-c", "--critical", type=int, help="Critical threshold.") @click.pass_context def cli(ctx, warning, critical): """Main command for Icinga plugins """ ctx.ensure_object(dict) if warning: - ctx.obj['warning_threshold'] = int(warning) + ctx.obj["warning_threshold"] = int(warning) if critical: - ctx.obj['critical_threshold'] = int(critical) + ctx.obj["critical_threshold"] = int(critical) -@cli.group(name='check-vault') -@click.option('--swh-storage-url', type=str, required=True, - help='URL to an swh-storage HTTP API') -@click.option('--swh-web-url', type=str, required=True, - help='URL to an swh-web instance') -@click.option('--poll-interval', type=int, default=10, - help='Interval (in seconds) between two polls to the API, ' - 'to check for cooking status.') +@cli.group(name="check-vault") +@click.option( + "--swh-storage-url", type=str, required=True, help="URL to an swh-storage HTTP API" +) +@click.option( + "--swh-web-url", type=str, required=True, help="URL to an swh-web instance" +) +@click.option( + "--poll-interval", + type=int, + default=10, + help="Interval (in seconds) between two polls to the API, " + "to check for cooking status.", +) @click.pass_context def check_vault(ctx, **kwargs): ctx.obj.update(kwargs) -@check_vault.command(name='directory') +@check_vault.command(name="directory") @click.pass_context def check_vault_directory(ctx): """Picks a random directory, requests its cooking via swh-web, and waits for completion.""" from .vault import VaultCheck + sys.exit(VaultCheck(ctx.obj).main()) -@cli.group(name='check-deposit') -@click.option('--server', type=str, - default='https://deposit.softwareheritage.org/1', - help='URL to the SWORD server to test') -@click.option('--username', type=str, required=True, - help='Login for the SWORD server') -@click.option('--password', type=str, required=True, - help='Password for the SWORD server') -@click.option('--collection', type=str, required=True, - help='Software collection to use on the SWORD server') -@click.option('--poll-interval', type=int, default=10, - help='Interval (in seconds) between two polls to the API, ' - 'to check for ingestion status.') +@cli.group(name="check-deposit") +@click.option( + "--server", + type=str, + default="https://deposit.softwareheritage.org/1", + help="URL to the SWORD server to test", +) +@click.option("--username", type=str, required=True, help="Login for the SWORD server") +@click.option( + "--password", type=str, required=True, help="Password for the SWORD server" +) +@click.option( + "--collection", + type=str, + required=True, + help="Software collection to use on the SWORD server", +) +@click.option( + "--poll-interval", + type=int, + default=10, + help="Interval (in seconds) between two polls to the API, " + "to check for ingestion status.", +) @click.pass_context def check_deposit(ctx, **kwargs): ctx.obj.update(kwargs) -@check_deposit.command(name='single') -@click.option('--archive', type=click.Path(), required=True, - help='Software artefact to upload') -@click.option('--metadata', type=click.Path(), required=True, - help='Metadata file for the software artefact.') +@check_deposit.command(name="single") +@click.option( + "--archive", type=click.Path(), required=True, help="Software artefact to upload" +) +@click.option( + "--metadata", + type=click.Path(), + required=True, + help="Metadata file for the software artefact.", +) @click.pass_context def check_deposit_single(ctx, **kwargs): """Checks the provided archive and metadata file and be deposited.""" diff --git a/swh/icinga_plugins/deposit.py b/swh/icinga_plugins/deposit.py --- a/swh/icinga_plugins/deposit.py +++ b/swh/icinga_plugins/deposit.py @@ -13,49 +13,52 @@ class DepositCheck(BaseCheck): - TYPE = 'DEPOSIT' + TYPE = "DEPOSIT" DEFAULT_WARNING_THRESHOLD = 120 DEFAULT_CRITICAL_THRESHOLD = 3600 def __init__(self, obj): super().__init__(obj) - self._poll_interval = obj['poll_interval'] - self._archive_path = obj['archive'] - self._metadata_path = obj['metadata'] - self._collection = obj['collection'] - - self._client = PublicApiDepositClient({ - 'url': obj['server'], - 'auth': { - 'username': obj['username'], - 'password': obj['password'], - }, - }) + self._poll_interval = obj["poll_interval"] + self._archive_path = obj["archive"] + self._metadata_path = obj["metadata"] + self._collection = obj["collection"] + + self._client = PublicApiDepositClient( + { + "url": obj["server"], + "auth": {"username": obj["username"], "password": obj["password"]}, + } + ) def upload_deposit(self): result = self._client.deposit_create( - archive=self._archive_path, metadata=self._metadata_path, - collection=self._collection, in_progress=False, - slug='check-deposit-%s' % datetime.datetime.now().isoformat()) - self._deposit_id = result['deposit_id'] + archive=self._archive_path, + metadata=self._metadata_path, + collection=self._collection, + in_progress=False, + slug="check-deposit-%s" % datetime.datetime.now().isoformat(), + ) + self._deposit_id = result["deposit_id"] return result def get_deposit_status(self): return self._client.deposit_status( - collection=self._collection, - deposit_id=self._deposit_id) + collection=self._collection, deposit_id=self._deposit_id + ) def wait_while_status(self, statuses, start_time, metrics, result): - while result['deposit_status'] in statuses: - metrics['total_time'] = time.time() - start_time - if metrics['total_time'] > self.critical_threshold: + while result["deposit_status"] in statuses: + metrics["total_time"] = time.time() - start_time + if metrics["total_time"] > self.critical_threshold: self.print_result( - 'CRITICAL', - f'Timed out while in status ' + "CRITICAL", + f"Timed out while in status " f'{result["deposit_status"]} ' f'({metrics["total_time"]}s seconds since deposit ' - f'started)', - **metrics) + f"started)", + **metrics, + ) sys.exit(2) time.sleep(self._poll_interval) @@ -70,53 +73,56 @@ # Upload the archive and metadata result = self.upload_deposit() - metrics['upload_time'] = time.time() - start_time + metrics["upload_time"] = time.time() - start_time # Wait for validation - result = self.wait_while_status( - ['deposited'], start_time, metrics, result) - metrics['total_time'] = time.time() - start_time - metrics['validation_time'] = \ - metrics['total_time'] - metrics['upload_time'] + result = self.wait_while_status(["deposited"], start_time, metrics, result) + metrics["total_time"] = time.time() - start_time + metrics["validation_time"] = metrics["total_time"] - metrics["upload_time"] # Check validation succeeded - if result['deposit_status'] == 'rejected': + if result["deposit_status"] == "rejected": self.print_result( - 'CRITICAL', + "CRITICAL", f'Deposit was rejected: {result["deposit_status_detail"]}', - **metrics) + **metrics, + ) return 2 # Wait for loading result = self.wait_while_status( - ['verified', 'loading'], start_time, metrics, result) - metrics['total_time'] = time.time() - start_time - metrics['load_time'] = ( - metrics['total_time'] - metrics['upload_time'] - - metrics['validation_time']) + ["verified", "loading"], start_time, metrics, result + ) + metrics["total_time"] = time.time() - start_time + metrics["load_time"] = ( + metrics["total_time"] - metrics["upload_time"] - metrics["validation_time"] + ) # Check loading succeeded - if result['deposit_status'] == 'failed': + if result["deposit_status"] == "failed": self.print_result( - 'CRITICAL', + "CRITICAL", f'Deposit loading failed: {result["deposit_status_detail"]}', - **metrics) + **metrics, + ) return 2 # Check for unexpected status - if result['deposit_status'] != 'done': + if result["deposit_status"] != "done": self.print_result( - 'CRITICAL', + "CRITICAL", f'Deposit got unexpected status: {result["deposit_status"]} ' f'({result["deposit_status_detail"]})', - **metrics) + **metrics, + ) return 2 # Everything went fine, check total time wasn't too large and # print result - (status_code, status) = self.get_status(metrics['total_time']) + (status_code, status) = self.get_status(metrics["total_time"]) self.print_result( status, f'Deposit took {metrics["total_time"]:.2f}s and succeeded.', - **metrics) + **metrics, + ) return status_code diff --git a/swh/icinga_plugins/tests/conftest.py b/swh/icinga_plugins/tests/conftest.py --- a/swh/icinga_plugins/tests/conftest.py +++ b/swh/icinga_plugins/tests/conftest.py @@ -21,5 +21,5 @@ def fake_time(): return start_time + time_offset - mocker.patch('time.sleep', side_effect=fake_sleep) - mocker.patch('time.time', side_effect=fake_time) + mocker.patch("time.sleep", side_effect=fake_sleep) + mocker.patch("time.time", side_effect=fake_time) diff --git a/swh/icinga_plugins/tests/test_deposit.py b/swh/icinga_plugins/tests/test_deposit.py --- a/swh/icinga_plugins/tests/test_deposit.py +++ b/swh/icinga_plugins/tests/test_deposit.py @@ -15,17 +15,21 @@ from .web_scenario import WebScenario -BASE_URL = 'http://swh-deposit.example.org/1' +BASE_URL = "http://swh-deposit.example.org/1" COMMON_OPTIONS = [ - '--server', BASE_URL, - '--username', 'test', - '--password', 'test', - '--collection', 'testcol', + "--server", + BASE_URL, + "--username", + "test", + "--password", + "test", + "--collection", + "testcol", ] -SAMPLE_METADATA = ''' +SAMPLE_METADATA = """ @@ -36,10 +40,10 @@ No One -''' +""" -ENTRY_TEMPLATE = ''' +ENTRY_TEMPLATE = """ @@ -50,10 +54,10 @@ http://purl.org/net/sword/package/SimpleZip -''' +""" -STATUS_TEMPLATE = ''' +STATUS_TEMPLATE = """ @@ -61,38 +65,36 @@ {status} {status_detail} -''' +""" -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def tmp_path(tmp_path_factory): return tmp_path_factory.mktemp(__name__) -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def sample_metadata(tmp_path): """Returns a sample metadata file's path """ - path = os.path.join(tmp_path, 'metadata.xml') + path = os.path.join(tmp_path, "metadata.xml") - with open(path, 'w') as fd: + with open(path, "w") as fd: fd.write(SAMPLE_METADATA) return path -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def sample_archive(tmp_path): """Returns a sample archive's path """ - path = os.path.join(tmp_path, 'archive.tar.gz') + path = os.path.join(tmp_path, "archive.tar.gz") - with tarfile.open(path, 'w:gz') as tf: - tf.addfile( - tarfile.TarInfo('hello.py'), - io.BytesIO(b'print("Hello world")')) + with tarfile.open(path, "w:gz") as tf: + tf.addfile(tarfile.TarInfo("hello.py"), io.BytesIO(b'print("Hello world")')) return path @@ -107,269 +109,359 @@ def test_deposit_immediate_success( - requests_mock, mocker, sample_archive, sample_metadata, mocked_time): + requests_mock, mocker, sample_archive, sample_metadata, mocked_time +): scenario = WebScenario() scenario.add_step( - 'post', BASE_URL + '/testcol/', - ENTRY_TEMPLATE.format(status='done')) + "post", BASE_URL + "/testcol/", ENTRY_TEMPLATE.format(status="done") + ) scenario.install_mock(requests_mock) - result = invoke([ - 'check-deposit', - *COMMON_OPTIONS, - 'single', - '--archive', sample_archive, - '--metadata', sample_metadata, - ]) + result = invoke( + [ + "check-deposit", + *COMMON_OPTIONS, + "single", + "--archive", + sample_archive, + "--metadata", + sample_metadata, + ] + ) assert result.output == ( "DEPOSIT OK - Deposit took 0.00s and succeeded.\n" "| 'load_time' = 0.00s\n" "| 'total_time' = 0.00s\n" "| 'upload_time' = 0.00s\n" - "| 'validation_time' = 0.00s\n") + "| 'validation_time' = 0.00s\n" + ) assert result.exit_code == 0, result.output def test_deposit_delays( - requests_mock, mocker, sample_archive, sample_metadata, mocked_time): + requests_mock, mocker, sample_archive, sample_metadata, mocked_time +): scenario = WebScenario() scenario.add_step( - 'post', BASE_URL + '/testcol/', - ENTRY_TEMPLATE.format(status='deposited')) + "post", BASE_URL + "/testcol/", ENTRY_TEMPLATE.format(status="deposited") + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='verified', status_detail='')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="verified", status_detail=""), + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='loading', status_detail='')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="loading", status_detail=""), + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='done', status_detail='')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="done", status_detail=""), + ) scenario.install_mock(requests_mock) - result = invoke([ - 'check-deposit', - *COMMON_OPTIONS, - 'single', - '--archive', sample_archive, - '--metadata', sample_metadata, - ]) + result = invoke( + [ + "check-deposit", + *COMMON_OPTIONS, + "single", + "--archive", + sample_archive, + "--metadata", + sample_metadata, + ] + ) assert result.output == ( "DEPOSIT OK - Deposit took 30.00s and succeeded.\n" "| 'load_time' = 20.00s\n" "| 'total_time' = 30.00s\n" "| 'upload_time' = 0.00s\n" - "| 'validation_time' = 10.00s\n") + "| 'validation_time' = 10.00s\n" + ) assert result.exit_code == 0, result.output def test_deposit_delay_warning( - requests_mock, mocker, sample_archive, sample_metadata, mocked_time): + requests_mock, mocker, sample_archive, sample_metadata, mocked_time +): scenario = WebScenario() scenario.add_step( - 'post', BASE_URL + '/testcol/', - ENTRY_TEMPLATE.format(status='deposited')) + "post", BASE_URL + "/testcol/", ENTRY_TEMPLATE.format(status="deposited") + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='verified', status_detail='')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="verified", status_detail=""), + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='done', status_detail='')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="done", status_detail=""), + ) scenario.install_mock(requests_mock) - result = invoke([ - '--warning', '15', - 'check-deposit', - *COMMON_OPTIONS, - 'single', - '--archive', sample_archive, - '--metadata', sample_metadata, - ], catch_exceptions=True) + result = invoke( + [ + "--warning", + "15", + "check-deposit", + *COMMON_OPTIONS, + "single", + "--archive", + sample_archive, + "--metadata", + sample_metadata, + ], + catch_exceptions=True, + ) assert result.output == ( "DEPOSIT WARNING - Deposit took 20.00s and succeeded.\n" "| 'load_time' = 10.00s\n" "| 'total_time' = 20.00s\n" "| 'upload_time' = 0.00s\n" - "| 'validation_time' = 10.00s\n") + "| 'validation_time' = 10.00s\n" + ) assert result.exit_code == 1, result.output def test_deposit_delay_critical( - requests_mock, mocker, sample_archive, sample_metadata, mocked_time): + requests_mock, mocker, sample_archive, sample_metadata, mocked_time +): scenario = WebScenario() scenario.add_step( - 'post', BASE_URL + '/testcol/', - ENTRY_TEMPLATE.format(status='deposited')) + "post", BASE_URL + "/testcol/", ENTRY_TEMPLATE.format(status="deposited") + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='verified', status_detail='')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="verified", status_detail=""), + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='done', status_detail=''), - callback=lambda: time.sleep(60)) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="done", status_detail=""), + callback=lambda: time.sleep(60), + ) scenario.install_mock(requests_mock) - result = invoke([ - '--critical', '50', - 'check-deposit', - *COMMON_OPTIONS, - 'single', - '--archive', sample_archive, - '--metadata', sample_metadata, - ], catch_exceptions=True) + result = invoke( + [ + "--critical", + "50", + "check-deposit", + *COMMON_OPTIONS, + "single", + "--archive", + sample_archive, + "--metadata", + sample_metadata, + ], + catch_exceptions=True, + ) assert result.output == ( "DEPOSIT CRITICAL - Deposit took 80.00s and succeeded.\n" "| 'load_time' = 70.00s\n" "| 'total_time' = 80.00s\n" "| 'upload_time' = 0.00s\n" - "| 'validation_time' = 10.00s\n") + "| 'validation_time' = 10.00s\n" + ) assert result.exit_code == 2, result.output def test_deposit_timeout( - requests_mock, mocker, sample_archive, sample_metadata, mocked_time): + requests_mock, mocker, sample_archive, sample_metadata, mocked_time +): scenario = WebScenario() scenario.add_step( - 'post', BASE_URL + '/testcol/', - ENTRY_TEMPLATE.format(status='deposited'), - callback=lambda: time.sleep(1500)) + "post", + BASE_URL + "/testcol/", + ENTRY_TEMPLATE.format(status="deposited"), + callback=lambda: time.sleep(1500), + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='verified', status_detail=''), - callback=lambda: time.sleep(1500)) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="verified", status_detail=""), + callback=lambda: time.sleep(1500), + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='loading', status_detail=''), - callback=lambda: time.sleep(1500)) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="loading", status_detail=""), + callback=lambda: time.sleep(1500), + ) scenario.install_mock(requests_mock) - result = invoke([ - 'check-deposit', - *COMMON_OPTIONS, - 'single', - '--archive', sample_archive, - '--metadata', sample_metadata, - ], catch_exceptions=True) + result = invoke( + [ + "check-deposit", + *COMMON_OPTIONS, + "single", + "--archive", + sample_archive, + "--metadata", + sample_metadata, + ], + catch_exceptions=True, + ) assert result.output == ( "DEPOSIT CRITICAL - Timed out while in status loading " "(4520.0s seconds since deposit started)\n" "| 'total_time' = 4520.00s\n" "| 'upload_time' = 1500.00s\n" - "| 'validation_time' = 1510.00s\n") + "| 'validation_time' = 1510.00s\n" + ) assert result.exit_code == 2, result.output def test_deposit_rejected( - requests_mock, mocker, sample_archive, sample_metadata, mocked_time): + requests_mock, mocker, sample_archive, sample_metadata, mocked_time +): scenario = WebScenario() scenario.add_step( - 'post', BASE_URL + '/testcol/', - ENTRY_TEMPLATE.format(status='deposited')) + "post", BASE_URL + "/testcol/", ENTRY_TEMPLATE.format(status="deposited") + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='rejected', status_detail='booo')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="rejected", status_detail="booo"), + ) scenario.install_mock(requests_mock) - result = invoke([ - 'check-deposit', - *COMMON_OPTIONS, - 'single', - '--archive', sample_archive, - '--metadata', sample_metadata, - ], catch_exceptions=True) + result = invoke( + [ + "check-deposit", + *COMMON_OPTIONS, + "single", + "--archive", + sample_archive, + "--metadata", + sample_metadata, + ], + catch_exceptions=True, + ) assert result.output == ( "DEPOSIT CRITICAL - Deposit was rejected: booo\n" "| 'total_time' = 10.00s\n" "| 'upload_time' = 0.00s\n" - "| 'validation_time' = 10.00s\n") + "| 'validation_time' = 10.00s\n" + ) assert result.exit_code == 2, result.output def test_deposit_failed( - requests_mock, mocker, sample_archive, sample_metadata, mocked_time): + requests_mock, mocker, sample_archive, sample_metadata, mocked_time +): scenario = WebScenario() scenario.add_step( - 'post', BASE_URL + '/testcol/', - ENTRY_TEMPLATE.format(status='deposited')) + "post", BASE_URL + "/testcol/", ENTRY_TEMPLATE.format(status="deposited") + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='verified', status_detail='')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="verified", status_detail=""), + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='loading', status_detail='')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="loading", status_detail=""), + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='failed', status_detail='booo')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="failed", status_detail="booo"), + ) scenario.install_mock(requests_mock) - result = invoke([ - 'check-deposit', - *COMMON_OPTIONS, - 'single', - '--archive', sample_archive, - '--metadata', sample_metadata, - ], catch_exceptions=True) + result = invoke( + [ + "check-deposit", + *COMMON_OPTIONS, + "single", + "--archive", + sample_archive, + "--metadata", + sample_metadata, + ], + catch_exceptions=True, + ) assert result.output == ( "DEPOSIT CRITICAL - Deposit loading failed: booo\n" "| 'load_time' = 20.00s\n" "| 'total_time' = 30.00s\n" "| 'upload_time' = 0.00s\n" - "| 'validation_time' = 10.00s\n") + "| 'validation_time' = 10.00s\n" + ) assert result.exit_code == 2, result.output def test_deposit_unexpected_status( - requests_mock, mocker, sample_archive, sample_metadata, mocked_time): + requests_mock, mocker, sample_archive, sample_metadata, mocked_time +): scenario = WebScenario() scenario.add_step( - 'post', BASE_URL + '/testcol/', - ENTRY_TEMPLATE.format(status='deposited')) + "post", BASE_URL + "/testcol/", ENTRY_TEMPLATE.format(status="deposited") + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='verified', status_detail='')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="verified", status_detail=""), + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='loading', status_detail='')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="loading", status_detail=""), + ) scenario.add_step( - 'get', BASE_URL + '/testcol/42/status/', - STATUS_TEMPLATE.format(status='what', status_detail='booo')) + "get", + BASE_URL + "/testcol/42/status/", + STATUS_TEMPLATE.format(status="what", status_detail="booo"), + ) scenario.install_mock(requests_mock) - result = invoke([ - 'check-deposit', - *COMMON_OPTIONS, - 'single', - '--archive', sample_archive, - '--metadata', sample_metadata, - ], catch_exceptions=True) + result = invoke( + [ + "check-deposit", + *COMMON_OPTIONS, + "single", + "--archive", + sample_archive, + "--metadata", + sample_metadata, + ], + catch_exceptions=True, + ) assert result.output == ( "DEPOSIT CRITICAL - Deposit got unexpected status: what (booo)\n" "| 'load_time' = 20.00s\n" "| 'total_time' = 30.00s\n" "| 'upload_time' = 0.00s\n" - "| 'validation_time' = 10.00s\n") + "| 'validation_time' = 10.00s\n" + ) assert result.exit_code == 2, result.output diff --git a/swh/icinga_plugins/tests/test_vault.py b/swh/icinga_plugins/tests/test_vault.py --- a/swh/icinga_plugins/tests/test_vault.py +++ b/swh/icinga_plugins/tests/test_vault.py @@ -11,13 +11,13 @@ from .web_scenario import WebScenario -dir_id = 'ab'*20 +dir_id = "ab" * 20 response_pending = { "obj_id": dir_id, "obj_type": "directory", "progress_message": "foo", - "status": "pending" + "status": "pending", } response_done = { @@ -25,21 +25,21 @@ "id": 9, "obj_id": dir_id, "obj_type": "directory", - "status": "done" + "status": "done", } response_failed = { "obj_id": dir_id, "obj_type": "directory", "progress_message": "foobar", - "status": "failed" + "status": "failed", } response_unknown_status = { "obj_id": dir_id, "obj_type": "directory", "progress_message": "what", - "status": "boo" + "status": "boo", } @@ -63,143 +63,170 @@ def test_vault_immediate_success(requests_mock, mocker, mocked_time): scenario = WebScenario() - url = f'mock://swh-web.example.org/api/1/vault/directory/{dir_id}/' + url = f"mock://swh-web.example.org/api/1/vault/directory/{dir_id}/" - scenario.add_step('get', url, {}, status_code=404) - scenario.add_step('post', url, response_pending) - scenario.add_step('get', url, response_done) + scenario.add_step("get", url, {}, status_code=404) + scenario.add_step("post", url, response_pending) + scenario.add_step("get", url, response_done) scenario.install_mock(requests_mock) - get_storage_mock = mocker.patch('swh.icinga_plugins.vault.get_storage') + get_storage_mock = mocker.patch("swh.icinga_plugins.vault.get_storage") get_storage_mock.side_effect = FakeStorage - result = invoke([ - 'check-vault', - '--swh-web-url', 'mock://swh-web.example.org', - '--swh-storage-url', 'foo://example.org', - 'directory', - ]) + result = invoke( + [ + "check-vault", + "--swh-web-url", + "mock://swh-web.example.org", + "--swh-storage-url", + "foo://example.org", + "directory", + ] + ) assert result.output == ( f"VAULT OK - cooking directory {dir_id} took " f"10.00s and succeeded.\n" - f"| 'total_time' = 10.00s\n") + f"| 'total_time' = 10.00s\n" + ) assert result.exit_code == 0, result.output def test_vault_delayed_success(requests_mock, mocker, mocked_time): scenario = WebScenario() - url = f'mock://swh-web.example.org/api/1/vault/directory/{dir_id}/' + url = f"mock://swh-web.example.org/api/1/vault/directory/{dir_id}/" - scenario.add_step('get', url, {}, status_code=404) - scenario.add_step('post', url, response_pending) - scenario.add_step('get', url, response_pending) - scenario.add_step('get', url, response_done) + scenario.add_step("get", url, {}, status_code=404) + scenario.add_step("post", url, response_pending) + scenario.add_step("get", url, response_pending) + scenario.add_step("get", url, response_done) scenario.install_mock(requests_mock) - get_storage_mock = mocker.patch('swh.icinga_plugins.vault.get_storage') + get_storage_mock = mocker.patch("swh.icinga_plugins.vault.get_storage") get_storage_mock.side_effect = FakeStorage - result = invoke([ - 'check-vault', - '--swh-web-url', 'mock://swh-web.example.org', - '--swh-storage-url', 'foo://example.org', - 'directory', - ]) + result = invoke( + [ + "check-vault", + "--swh-web-url", + "mock://swh-web.example.org", + "--swh-storage-url", + "foo://example.org", + "directory", + ] + ) assert result.output == ( f"VAULT OK - cooking directory {dir_id} took " f"20.00s and succeeded.\n" - f"| 'total_time' = 20.00s\n") + f"| 'total_time' = 20.00s\n" + ) assert result.exit_code == 0, result.output def test_vault_failure(requests_mock, mocker, mocked_time): scenario = WebScenario() - url = f'mock://swh-web.example.org/api/1/vault/directory/{dir_id}/' + url = f"mock://swh-web.example.org/api/1/vault/directory/{dir_id}/" - scenario.add_step('get', url, {}, status_code=404) - scenario.add_step('post', url, response_pending) - scenario.add_step('get', url, response_failed) + scenario.add_step("get", url, {}, status_code=404) + scenario.add_step("post", url, response_pending) + scenario.add_step("get", url, response_failed) scenario.install_mock(requests_mock) - get_storage_mock = mocker.patch('swh.icinga_plugins.vault.get_storage') + get_storage_mock = mocker.patch("swh.icinga_plugins.vault.get_storage") get_storage_mock.side_effect = FakeStorage - result = invoke([ - 'check-vault', - '--swh-web-url', 'mock://swh-web.example.org', - '--swh-storage-url', 'foo://example.org', - 'directory', - ], catch_exceptions=True) + result = invoke( + [ + "check-vault", + "--swh-web-url", + "mock://swh-web.example.org", + "--swh-storage-url", + "foo://example.org", + "directory", + ], + catch_exceptions=True, + ) assert result.output == ( f"VAULT CRITICAL - cooking directory {dir_id} took " f"10.00s and failed with: foobar\n" - f"| 'total_time' = 10.00s\n") + f"| 'total_time' = 10.00s\n" + ) assert result.exit_code == 2, result.output def test_vault_unknown_status(requests_mock, mocker, mocked_time): scenario = WebScenario() - url = f'mock://swh-web.example.org/api/1/vault/directory/{dir_id}/' + url = f"mock://swh-web.example.org/api/1/vault/directory/{dir_id}/" - scenario.add_step('get', url, {}, status_code=404) - scenario.add_step('post', url, response_pending) - scenario.add_step('get', url, response_unknown_status) + scenario.add_step("get", url, {}, status_code=404) + scenario.add_step("post", url, response_pending) + scenario.add_step("get", url, response_unknown_status) scenario.install_mock(requests_mock) - get_storage_mock = mocker.patch('swh.icinga_plugins.vault.get_storage') + get_storage_mock = mocker.patch("swh.icinga_plugins.vault.get_storage") get_storage_mock.side_effect = FakeStorage - result = invoke([ - 'check-vault', - '--swh-web-url', 'mock://swh-web.example.org', - '--swh-storage-url', 'foo://example.org', - 'directory', - ], catch_exceptions=True) + result = invoke( + [ + "check-vault", + "--swh-web-url", + "mock://swh-web.example.org", + "--swh-storage-url", + "foo://example.org", + "directory", + ], + catch_exceptions=True, + ) assert result.output == ( f"VAULT CRITICAL - cooking directory {dir_id} took " f"10.00s and resulted in unknown status: boo\n" - f"| 'total_time' = 10.00s\n") + f"| 'total_time' = 10.00s\n" + ) assert result.exit_code == 2, result.output def test_vault_timeout(requests_mock, mocker, mocked_time): scenario = WebScenario() - url = f'mock://swh-web.example.org/api/1/vault/directory/{dir_id}/' + url = f"mock://swh-web.example.org/api/1/vault/directory/{dir_id}/" - scenario.add_step('get', url, {}, status_code=404) - scenario.add_step('post', url, response_pending) - scenario.add_step('get', url, response_pending) - scenario.add_step('get', url, response_pending, - callback=lambda: time.sleep(4000)) + scenario.add_step("get", url, {}, status_code=404) + scenario.add_step("post", url, response_pending) + scenario.add_step("get", url, response_pending) + scenario.add_step("get", url, response_pending, callback=lambda: time.sleep(4000)) scenario.install_mock(requests_mock) - get_storage_mock = mocker.patch('swh.icinga_plugins.vault.get_storage') + get_storage_mock = mocker.patch("swh.icinga_plugins.vault.get_storage") get_storage_mock.side_effect = FakeStorage - result = invoke([ - 'check-vault', - '--swh-web-url', 'mock://swh-web.example.org', - '--swh-storage-url', 'foo://example.org', - 'directory', - ], catch_exceptions=True) + result = invoke( + [ + "check-vault", + "--swh-web-url", + "mock://swh-web.example.org", + "--swh-storage-url", + "foo://example.org", + "directory", + ], + catch_exceptions=True, + ) assert result.output == ( f"VAULT CRITICAL - cooking directory {dir_id} took more than " f"4020.00s and has status: foo\n" - f"| 'total_time' = 4020.00s\n") + f"| 'total_time' = 4020.00s\n" + ) assert result.exit_code == 2, result.output @@ -208,29 +235,34 @@ test that vault_check requests another one.""" scenario = WebScenario() - url = f'mock://swh-web.example.org/api/1/vault/directory/{dir_id}/' + url = f"mock://swh-web.example.org/api/1/vault/directory/{dir_id}/" - scenario.add_step('get', url, {}, status_code=200) - scenario.add_step('get', url, {}, status_code=404) - scenario.add_step('post', url, response_pending) - scenario.add_step('get', url, response_done) + scenario.add_step("get", url, {}, status_code=200) + scenario.add_step("get", url, {}, status_code=404) + scenario.add_step("post", url, response_pending) + scenario.add_step("get", url, response_done) scenario.install_mock(requests_mock) - get_storage_mock = mocker.patch('swh.icinga_plugins.vault.get_storage') + get_storage_mock = mocker.patch("swh.icinga_plugins.vault.get_storage") get_storage_mock.side_effect = FakeStorage - result = invoke([ - 'check-vault', - '--swh-web-url', 'mock://swh-web.example.org', - '--swh-storage-url', 'foo://example.org', - 'directory', - ]) + result = invoke( + [ + "check-vault", + "--swh-web-url", + "mock://swh-web.example.org", + "--swh-storage-url", + "foo://example.org", + "directory", + ] + ) assert result.output == ( f"VAULT OK - cooking directory {dir_id} took " f"10.00s and succeeded.\n" - f"| 'total_time' = 10.00s\n") + f"| 'total_time' = 10.00s\n" + ) assert result.exit_code == 0, result.output @@ -239,18 +271,21 @@ scenario = WebScenario() scenario.install_mock(requests_mock) - get_storage_mock = mocker.patch('swh.icinga_plugins.vault.get_storage') + get_storage_mock = mocker.patch("swh.icinga_plugins.vault.get_storage") get_storage_mock.side_effect = FakeStorage - mocker.patch( - f'{__name__}.FakeStorage.directory_get_random', return_value=None) - - result = invoke([ - 'check-vault', - '--swh-web-url', 'mock://swh-web.example.org', - '--swh-storage-url', 'foo://example.org', - 'directory', - ], catch_exceptions=True) - - assert result.output == ( - "VAULT CRITICAL - No directory exists in the archive.\n") + mocker.patch(f"{__name__}.FakeStorage.directory_get_random", return_value=None) + + result = invoke( + [ + "check-vault", + "--swh-web-url", + "mock://swh-web.example.org", + "--swh-storage-url", + "foo://example.org", + "directory", + ], + catch_exceptions=True, + ) + + assert result.output == ("VAULT CRITICAL - No directory exists in the archive.\n") assert result.exit_code == 2, result.output diff --git a/swh/icinga_plugins/tests/web_scenario.py b/swh/icinga_plugins/tests/web_scenario.py --- a/swh/icinga_plugins/tests/web_scenario.py +++ b/swh/icinga_plugins/tests/web_scenario.py @@ -68,8 +68,8 @@ """ for endpoint in self._endpoints: mocker.register_uri( - endpoint.method.upper(), endpoint.url, - text=self._request_callback) + endpoint.method.upper(), endpoint.url, text=self._request_callback + ) def _request_callback(self, request, context): step = self._steps[self._current_step] diff --git a/swh/icinga_plugins/vault.py b/swh/icinga_plugins/vault.py --- a/swh/icinga_plugins/vault.py +++ b/swh/icinga_plugins/vault.py @@ -17,18 +17,18 @@ class VaultCheck(BaseCheck): - TYPE = 'VAULT' + TYPE = "VAULT" DEFAULT_WARNING_THRESHOLD = 0 DEFAULT_CRITICAL_THRESHOLD = 3600 def __init__(self, obj): super().__init__(obj) - self._swh_storage = get_storage('remote', url=obj['swh_storage_url']) - self._swh_web_url = obj['swh_web_url'] - self._poll_interval = obj['poll_interval'] + self._swh_storage = get_storage("remote", url=obj["swh_storage_url"]) + self._swh_web_url = obj["swh_web_url"] + self._poll_interval = obj["poll_interval"] def _url_for_dir(self, dir_id): - return self._swh_web_url + f'/api/1/vault/directory/{dir_id.hex()}/' + return self._swh_web_url + f"/api/1/vault/directory/{dir_id.hex()}/" def _pick_directory(self): dir_ = self._swh_storage.directory_get_random() @@ -47,9 +47,7 @@ try: dir_id = self._pick_uncached_directory() except NoDirectory: - self.print_result( - 'CRITICAL', - 'No directory exists in the archive.') + self.print_result("CRITICAL", "No directory exists in the archive.") return 2 start_time = time.time() @@ -57,7 +55,7 @@ response = requests.post(self._url_for_dir(dir_id)) assert response.status_code == 200, (response, response.text) result = response.json() - while result['status'] in ('new', 'pending'): + while result["status"] in ("new", "pending"): time.sleep(self._poll_interval) response = requests.get(self._url_for_dir(dir_id)) assert response.status_code == 200, (response, response.text) @@ -67,32 +65,36 @@ if total_time > self.critical_threshold: self.print_result( - 'CRITICAL', - f'cooking directory {dir_id.hex()} took more than ' - f'{total_time:.2f}s and has status: ' + "CRITICAL", + f"cooking directory {dir_id.hex()} took more than " + f"{total_time:.2f}s and has status: " f'{result["progress_message"]}', - total_time=total_time) + total_time=total_time, + ) return 2 - if result['status'] == 'done': + if result["status"] == "done": (status_code, status) = self.get_status(total_time) self.print_result( status, - f'cooking directory {dir_id.hex()} took {total_time:.2f}s ' - f'and succeeded.', - total_time=total_time) + f"cooking directory {dir_id.hex()} took {total_time:.2f}s " + f"and succeeded.", + total_time=total_time, + ) return status_code - elif result['status'] == 'failed': + elif result["status"] == "failed": self.print_result( - 'CRITICAL', - f'cooking directory {dir_id.hex()} took {total_time:.2f}s ' + "CRITICAL", + f"cooking directory {dir_id.hex()} took {total_time:.2f}s " f'and failed with: {result["progress_message"]}', - total_time=total_time) + total_time=total_time, + ) return 2 else: self.print_result( - 'CRITICAL', - f'cooking directory {dir_id.hex()} took {total_time:.2f}s ' + "CRITICAL", + f"cooking directory {dir_id.hex()} took {total_time:.2f}s " f'and resulted in unknown status: {result["status"]}', - total_time=total_time) + total_time=total_time, + ) return 2 diff --git a/tox.ini b/tox.ini --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=flake8,mypy,py3 +envlist=black,flake8,mypy,py3 [testenv:py3] deps = @@ -11,6 +11,13 @@ --cov={envsitepackagesdir}/swh/icinga_plugins \ --cov-branch {posargs} +[testenv:black] +skip_install = true +deps = + black +commands = + {envpython} -m black --check swh + [testenv:flake8] skip_install = true deps =