diff --git a/requirements-swh.txt b/requirements-swh.txt
index bdf45e1..4abaae7 100644
--- a/requirements-swh.txt
+++ b/requirements-swh.txt
@@ -1 +1,3 @@
+swh.core[http] # required by swh.storage
+swh.deposit
swh.storage >= v0.0.162
diff --git a/swh/icinga_plugins/cli.py b/swh/icinga_plugins/cli.py
index a0e87e5..36d1546 100644
--- a/swh/icinga_plugins/cli.py
+++ b/swh/icinga_plugins/cli.py
@@ -1,46 +1,77 @@
# Copyright (C) 2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import sys
import click
from swh.core.cli import CONTEXT_SETTINGS
+from .deposit import DepositCheck
from .vault import VaultCheck
@click.group(name='icinga_plugins', context_settings=CONTEXT_SETTINGS)
@click.option('-w', '--warning', type=int,
help='Warning threshold.')
@click.option('-c', '--critical', type=int,
help='Critical threshold.')
@click.pass_context
def cli(ctx, **kwargs):
"""Main command for Icinga plugins
"""
ctx.ensure_object(dict)
ctx.obj.update(kwargs)
@cli.group(name='check-vault')
@click.option('--swh-storage-url', type=str, required=True,
help='URL to an swh-storage HTTP API')
@click.option('--swh-web-url', type=str, required=True,
help='URL to an swh-web instance')
@click.option('--poll-interval', type=int, default=10,
help='Interval (in seconds) between two polls to the API, '
'to check for cooking status.')
@click.pass_context
def check_vault(ctx, **kwargs):
ctx.obj.update(kwargs)
@check_vault.command(name='directory')
@click.pass_context
def check_vault_directory(ctx):
"""Picks a random directory, requests its cooking via swh-web,
and waits for completion."""
sys.exit(VaultCheck(ctx.obj).main())
+
+
+@cli.group(name='check-deposit')
+@click.option('--server', type=str,
+ default='https://deposit.softwareheritage.org/1',
+ help='URL to the SWORD server to test')
+@click.option('--username', type=str, required=True,
+ help='Login for the SWORD server')
+@click.option('--password', type=str, required=True,
+ help='Password for the SWORD server')
+@click.option('--collection', type=str, required=True,
+ help='Software collection to use on the SWORD server')
+@click.option('--poll-interval', type=int, default=10,
+ help='Interval (in seconds) between two polls to the API, '
+ 'to check for ingestion status.')
+@click.pass_context
+def check_deposit(ctx, **kwargs):
+ ctx.obj.update(kwargs)
+
+
+@check_deposit.command(name='single')
+@click.option('--archive', type=click.Path(), required=True,
+ help='Software artefact to upload')
+@click.option('--metadata', type=click.Path(), required=True,
+ help='Metadata file for the software artefact.')
+@click.pass_context
+def check_deposit_single(ctx, **kwargs):
+ """Checks the provided archive and metadata file and be deposited."""
+ ctx.obj.update(kwargs)
+ sys.exit(DepositCheck(ctx.obj).main())
diff --git a/swh/icinga_plugins/deposit.py b/swh/icinga_plugins/deposit.py
new file mode 100644
index 0000000..92fffe1
--- /dev/null
+++ b/swh/icinga_plugins/deposit.py
@@ -0,0 +1,122 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import datetime
+import sys
+import time
+
+from .base_check import BaseCheck
+
+from swh.deposit.client import PublicApiDepositClient
+
+
+class DepositCheck(BaseCheck):
+ TYPE = 'DEPOSIT'
+ DEFAULT_WARNING_THRESHOLD = 120
+ DEFAULT_CRITICAL_THRESHOLD = 3600
+
+ def __init__(self, obj):
+ super().__init__(obj)
+ self._poll_interval = obj['poll_interval']
+ self._archive_path = obj['archive']
+ self._metadata_path = obj['metadata']
+ self._collection = obj['collection']
+
+ self._client = PublicApiDepositClient({
+ 'url': obj['server'],
+ 'auth': {
+ 'username': obj['username'],
+ 'password': obj['password'],
+ },
+ })
+
+ def upload_deposit(self):
+ result = self._client.deposit_create(
+ archive=self._archive_path, metadata=self._metadata_path,
+ collection=self._collection, in_progress=False,
+ slug='check-deposit-%s' % datetime.datetime.now().isoformat())
+ self._deposit_id = result['deposit_id']
+ return result
+
+ def get_deposit_status(self):
+ return self._client.deposit_status(
+ collection=self._collection,
+ deposit_id=self._deposit_id)
+
+ def wait_while_status(self, statuses, start_time, metrics, result):
+ while result['deposit_status'] in statuses:
+ metrics['total_time'] = time.time() - start_time
+ if metrics['total_time'] > self.critical_threshold:
+ self.print_result(
+ 'CRITICAL',
+ f'Timed out while in status '
+ f'{result["deposit_status"]} '
+ f'({metrics["total_time"]}s seconds since deposit '
+ f'started)',
+ **metrics)
+ sys.exit(2)
+
+ time.sleep(self._poll_interval)
+
+ result = self.get_deposit_status()
+
+ return result
+
+ def main(self):
+ start_time = time.time()
+ metrics = {}
+
+ # Upload the archive and metadata
+ result = self.upload_deposit()
+ metrics['upload_time'] = time.time() - start_time
+
+ # Wait for validation
+ result = self.wait_while_status(
+ ['deposited'], start_time, metrics, result)
+ metrics['total_time'] = time.time() - start_time
+ metrics['validation_time'] = \
+ metrics['total_time'] - metrics['upload_time']
+
+ # Check validation succeeded
+ if result['deposit_status'] == 'rejected':
+ self.print_result(
+ 'CRITICAL',
+ f'Deposit was rejected: {result["deposit_status_detail"]}',
+ **metrics)
+ return 2
+
+ # Wait for loading
+ result = self.wait_while_status(
+ ['verified', 'loading'], start_time, metrics, result)
+ metrics['total_time'] = time.time() - start_time
+ metrics['load_time'] = (
+ metrics['total_time'] - metrics['upload_time']
+ - metrics['validation_time'])
+
+ # Check loading succeeded
+ if result['deposit_status'] == 'failed':
+ self.print_result(
+ 'CRITICAL',
+ f'Deposit loading failed: {result["deposit_status_detail"]}',
+ **metrics)
+ return 2
+
+ # Check for unexpected status
+ if result['deposit_status'] != 'done':
+ self.print_result(
+ 'CRITICAL',
+ f'Deposit got unexpected status: {result["deposit_status"]} '
+ f'({result["deposit_status_detail"]})',
+ **metrics)
+ return 2
+
+ # Everything went fine, check total time wasn't too large and
+ # print result
+ (status_code, status) = self.get_status(metrics['total_time'])
+ self.print_result(
+ status,
+ f'Deposit took {metrics["total_time"]:.2f}s and succeeded.',
+ **metrics)
+ return status_code
diff --git a/swh/icinga_plugins/tests/conftest.py b/swh/icinga_plugins/tests/conftest.py
new file mode 100644
index 0000000..860b69e
--- /dev/null
+++ b/swh/icinga_plugins/tests/conftest.py
@@ -0,0 +1,25 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import time
+
+import pytest
+
+
+@pytest.fixture
+def mocked_time(mocker):
+ start_time = time.time()
+
+ time_offset = 0
+
+ def fake_sleep(seconds):
+ nonlocal time_offset
+ time_offset += seconds
+
+ def fake_time():
+ return start_time + time_offset
+
+ mocker.patch('time.sleep', side_effect=fake_sleep)
+ mocker.patch('time.time', side_effect=fake_time)
diff --git a/swh/icinga_plugins/tests/test_deposit.py b/swh/icinga_plugins/tests/test_deposit.py
new file mode 100644
index 0000000..fd81e15
--- /dev/null
+++ b/swh/icinga_plugins/tests/test_deposit.py
@@ -0,0 +1,306 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import io
+import os
+import tarfile
+import time
+
+from click.testing import CliRunner
+import pytest
+
+from swh.icinga_plugins.cli import cli
+from .web_scenario import WebScenario
+
+
+BASE_URL = 'mock://swh-deposit.example.org/1'
+
+COMMON_OPTIONS = [
+ '--server', BASE_URL,
+ '--username', 'test',
+ '--password', 'test',
+ '--collection', 'testcol',
+]
+
+
+SAMPLE_METADATA = '''
+
+
+ Test Software
+ swh
+ test-software
+
+ No One
+
+
+'''
+
+
+ENTRY_TEMPLATE = '''
+
+ 42
+ 2019-12-19 18:11:00
+ foo.tar.gz
+ {status}
+
+ http://purl.org/net/sword/package/SimpleZip
+
+'''
+
+
+STATUS_TEMPLATE = '''
+
+ 42
+ {status}
+ {status_detail}
+
+'''
+
+
+@pytest.fixture(scope='session')
+def tmp_path(tmp_path_factory):
+ return tmp_path_factory.mktemp(__name__)
+
+
+@pytest.fixture(scope='session')
+def sample_metadata(tmp_path):
+ """Returns a sample metadata file's path
+
+ """
+ path = os.path.join(tmp_path, 'metadata.xml')
+
+ with open(path, 'w') as fd:
+ fd.write(SAMPLE_METADATA)
+
+ return path
+
+
+@pytest.fixture(scope='session')
+def sample_archive(tmp_path):
+ """Returns a sample archive's path
+
+ """
+ path = os.path.join(tmp_path, 'archive.tar.gz')
+
+ with tarfile.open(path, 'w:gz') as tf:
+ tf.addfile(
+ tarfile.TarInfo('hello.py'),
+ io.BytesIO(b'print("Hello world")'))
+
+ return path
+
+
+def invoke(args, catch_exceptions=False):
+ runner = CliRunner()
+ result = runner.invoke(cli, args)
+ if not catch_exceptions and result.exception:
+ print(result.output)
+ raise result.exception
+ return result
+
+
+def test_deposit_immediate_success(
+ requests_mock, mocker, sample_archive, sample_metadata, mocked_time):
+ scenario = WebScenario()
+
+ scenario.add_step(
+ 'post', BASE_URL + '/testcol/',
+ ENTRY_TEMPLATE.format(status='done'))
+
+ scenario.install_mock(requests_mock)
+
+ result = invoke([
+ 'check-deposit',
+ *COMMON_OPTIONS,
+ 'single',
+ '--archive', sample_archive,
+ '--metadata', sample_metadata,
+ ])
+
+ assert result.output == (
+ "DEPOSIT OK - Deposit took 0.00s and succeeded.\n"
+ "| 'load_time' = 0.00s\n"
+ "| 'total_time' = 0.00s\n"
+ "| 'upload_time' = 0.00s\n"
+ "| 'validation_time' = 0.00s\n")
+ assert result.exit_code == 0, result.output
+
+
+def test_deposit_delays(
+ requests_mock, mocker, sample_archive, sample_metadata, mocked_time):
+ scenario = WebScenario()
+
+ scenario.add_step(
+ 'post', BASE_URL + '/testcol/',
+ ENTRY_TEMPLATE.format(status='deposited'))
+ scenario.add_step(
+ 'get', BASE_URL + '/testcol/42/status/',
+ STATUS_TEMPLATE.format(status='verified', status_detail=''))
+ scenario.add_step(
+ 'get', BASE_URL + '/testcol/42/status/',
+ STATUS_TEMPLATE.format(status='loading', status_detail=''))
+ scenario.add_step(
+ 'get', BASE_URL + '/testcol/42/status/',
+ STATUS_TEMPLATE.format(status='done', status_detail=''))
+
+ scenario.install_mock(requests_mock)
+
+ result = invoke([
+ 'check-deposit',
+ *COMMON_OPTIONS,
+ 'single',
+ '--archive', sample_archive,
+ '--metadata', sample_metadata,
+ ])
+
+ assert result.output == (
+ "DEPOSIT OK - Deposit took 30.00s and succeeded.\n"
+ "| 'load_time' = 20.00s\n"
+ "| 'total_time' = 30.00s\n"
+ "| 'upload_time' = 0.00s\n"
+ "| 'validation_time' = 10.00s\n")
+ assert result.exit_code == 0, result.output
+
+
+def test_deposit_timeout(
+ requests_mock, mocker, sample_archive, sample_metadata, mocked_time):
+ scenario = WebScenario()
+
+ scenario.add_step(
+ 'post', BASE_URL + '/testcol/',
+ ENTRY_TEMPLATE.format(status='deposited'),
+ callback=lambda: time.sleep(1500))
+ scenario.add_step(
+ 'get', BASE_URL + '/testcol/42/status/',
+ STATUS_TEMPLATE.format(status='verified', status_detail=''),
+ callback=lambda: time.sleep(1500))
+ scenario.add_step(
+ 'get', BASE_URL + '/testcol/42/status/',
+ STATUS_TEMPLATE.format(status='loading', status_detail=''),
+ callback=lambda: time.sleep(1500))
+
+ scenario.install_mock(requests_mock)
+
+ result = invoke([
+ 'check-deposit',
+ *COMMON_OPTIONS,
+ 'single',
+ '--archive', sample_archive,
+ '--metadata', sample_metadata,
+ ], catch_exceptions=True)
+
+ assert result.output == (
+ "DEPOSIT CRITICAL - Timed out while in status loading "
+ "(4520.0s seconds since deposit started)\n"
+ "| 'total_time' = 4520.00s\n"
+ "| 'upload_time' = 1500.00s\n"
+ "| 'validation_time' = 1510.00s\n")
+ assert result.exit_code == 2, result.output
+
+
+def test_deposit_rejected(
+ requests_mock, mocker, sample_archive, sample_metadata, mocked_time):
+ scenario = WebScenario()
+
+ scenario.add_step(
+ 'post', BASE_URL + '/testcol/',
+ ENTRY_TEMPLATE.format(status='deposited'))
+ scenario.add_step(
+ 'get', BASE_URL + '/testcol/42/status/',
+ STATUS_TEMPLATE.format(status='rejected', status_detail='booo'))
+
+ scenario.install_mock(requests_mock)
+
+ result = invoke([
+ 'check-deposit',
+ *COMMON_OPTIONS,
+ 'single',
+ '--archive', sample_archive,
+ '--metadata', sample_metadata,
+ ], catch_exceptions=True)
+
+ assert result.output == (
+ "DEPOSIT CRITICAL - Deposit was rejected: booo\n"
+ "| 'total_time' = 10.00s\n"
+ "| 'upload_time' = 0.00s\n"
+ "| 'validation_time' = 10.00s\n")
+ assert result.exit_code == 2, result.output
+
+
+def test_deposit_failed(
+ requests_mock, mocker, sample_archive, sample_metadata, mocked_time):
+ scenario = WebScenario()
+
+ scenario.add_step(
+ 'post', BASE_URL + '/testcol/',
+ ENTRY_TEMPLATE.format(status='deposited'))
+ scenario.add_step(
+ 'get', BASE_URL + '/testcol/42/status/',
+ STATUS_TEMPLATE.format(status='verified', status_detail=''))
+ scenario.add_step(
+ 'get', BASE_URL + '/testcol/42/status/',
+ STATUS_TEMPLATE.format(status='loading', status_detail=''))
+ scenario.add_step(
+ 'get', BASE_URL + '/testcol/42/status/',
+ STATUS_TEMPLATE.format(status='failed', status_detail='booo'))
+
+ scenario.install_mock(requests_mock)
+
+ result = invoke([
+ 'check-deposit',
+ *COMMON_OPTIONS,
+ 'single',
+ '--archive', sample_archive,
+ '--metadata', sample_metadata,
+ ], catch_exceptions=True)
+
+ assert result.output == (
+ "DEPOSIT CRITICAL - Deposit loading failed: booo\n"
+ "| 'load_time' = 20.00s\n"
+ "| 'total_time' = 30.00s\n"
+ "| 'upload_time' = 0.00s\n"
+ "| 'validation_time' = 10.00s\n")
+ assert result.exit_code == 2, result.output
+
+
+def test_deposit_unexpected_status(
+ requests_mock, mocker, sample_archive, sample_metadata, mocked_time):
+ scenario = WebScenario()
+
+ scenario.add_step(
+ 'post', BASE_URL + '/testcol/',
+ ENTRY_TEMPLATE.format(status='deposited'))
+ scenario.add_step(
+ 'get', BASE_URL + '/testcol/42/status/',
+ STATUS_TEMPLATE.format(status='verified', status_detail=''))
+ scenario.add_step(
+ 'get', BASE_URL + '/testcol/42/status/',
+ STATUS_TEMPLATE.format(status='loading', status_detail=''))
+ scenario.add_step(
+ 'get', BASE_URL + '/testcol/42/status/',
+ STATUS_TEMPLATE.format(status='what', status_detail='booo'))
+
+ scenario.install_mock(requests_mock)
+
+ result = invoke([
+ 'check-deposit',
+ *COMMON_OPTIONS,
+ 'single',
+ '--archive', sample_archive,
+ '--metadata', sample_metadata,
+ ], catch_exceptions=True)
+
+ assert result.output == (
+ "DEPOSIT CRITICAL - Deposit got unexpected status: what (booo)\n"
+ "| 'load_time' = 20.00s\n"
+ "| 'total_time' = 30.00s\n"
+ "| 'upload_time' = 0.00s\n"
+ "| 'validation_time' = 10.00s\n")
+ assert result.exit_code == 2, result.output
diff --git a/swh/icinga_plugins/tests/web_scenario.py b/swh/icinga_plugins/tests/web_scenario.py
index 57a31cc..0a0c57e 100644
--- a/swh/icinga_plugins/tests/web_scenario.py
+++ b/swh/icinga_plugins/tests/web_scenario.py
@@ -1,87 +1,90 @@
# Copyright (C) 2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""Wrapper around requests-mock to mock successive responses
from a web service.
Tests can build successive steps by calling :py:meth:`WebScenario.add_step`
with specifications of what endpoints should be called and in what order."""
from dataclasses import dataclass
import json
from typing import List, Set, Optional, Callable
import requests_mock
@dataclass(frozen=True)
class Step:
expected_method: str
expected_url: str
response: object
status_code: int = 200
callback: Optional[Callable[[], int]] = None
@dataclass(frozen=True)
class Endpoint:
method: str
url: str
class WebScenario:
"""Stores the state of the successive calls to the web service
expected by tests."""
_steps: List[Step]
_endpoints: Set[Endpoint]
_current_step: int
def __init__(self):
self._steps = []
self._endpoints = set()
self._current_step = 0
def add_endpoint(self, *args, **kwargs):
"""Adds an endpoint to be mocked.
Arguments are the same as :py:class:Endpoint.
"""
self._endpoints.add(Endpoint(*args, **kwargs))
def add_step(self, *args, **kwargs):
"""Adds an expected call to the list of expected calls.
Also automatically calls :py:meth:`add_endpoint` so the
associated endpoint is mocked.
Arguments are the same as :py:class:`Step`.
"""
step = Step(*args, **kwargs)
self._steps.append(step)
self.add_endpoint(step.expected_method, step.expected_url)
def install_mock(self, mocker: requests_mock.Mocker):
"""Mocks entrypoints registered with :py:meth:`add_endpoint`
(or :py:meth:`add_step`) using the provided mocker.
"""
for endpoint in self._endpoints:
mocker.register_uri(
endpoint.method.upper(), endpoint.url,
text=self._request_callback)
def _request_callback(self, request, context):
step = self._steps[self._current_step]
assert request.url == step.expected_url
assert request.method.upper() == step.expected_method.upper()
self._current_step += 1
context.status_code = step.status_code
if step.callback:
step.callback()
- return json.dumps(step.response)
+ if isinstance(step.response, str):
+ return step.response
+ else:
+ return json.dumps(step.response)