Page MenuHomeSoftware Heritage
Files F8395247

D2453.id8692.diff
No OneTemporary
Actions

D2453.id8692.diff
View Options

diff --git a/requirements-swh.txt b/requirements-swh.txt
--- a/requirements-swh.txt
+++ b/requirements-swh.txt
@@ -1 +1 @@
-# Add here internal Software Heritage dependencies, one per line.
+swh.storage >= v0.0.162
diff --git a/requirements-test.txt b/requirements-test.txt
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -1 +1,2 @@
pytest
+pytest-mock
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,3 @@
-# Add here external Python modules dependencies, one per line. Module names
-# should match https://pypi.python.org/pypi names. For the full spec or
-# dependency lines, see https://pip.readthedocs.org/en/1.1/requirements.html
+psycopg2
+requests
vcversioner
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -51,8 +51,8 @@
vcversioner={},
include_package_data=True,
entry_points='''
- [console_scripts]
- check_vault = swh.icinga_plugins.vault:main
+ [swh.cli.subcommands]
+ icinga_plugins=swh.icinga_plugins.cli:cli
''',
classifiers=[
"Programming Language :: Python :: 3",
diff --git a/swh/icinga_plugins/bar.py b/swh/icinga_plugins/bar.py
deleted file mode 100644
--- a/swh/icinga_plugins/bar.py
+++ /dev/null
@@ -1,4 +0,0 @@
-# Copyright (C) 2019 The Software Heritage developers
-# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
-# See top-level LICENSE file for more information
diff --git a/swh/icinga_plugins/cli.py b/swh/icinga_plugins/cli.py
--- a/swh/icinga_plugins/cli.py
+++ b/swh/icinga_plugins/cli.py
@@ -1,18 +1,43 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import sys
+
import click
from swh.core.cli import CONTEXT_SETTINGS
+from .vault import VaultCheck
+
-@click.group(name='foo', context_settings=CONTEXT_SETTINGS)
+@click.group(name='icinga_plugins', context_settings=CONTEXT_SETTINGS)
+@click.option('--swh-storage-url', type=str,
+ help='URL to an swh-storage HTTP API')
+@click.option('--swh-web-url', type=str,
+ help='URL to an swh-web instance')
@click.pass_context
-def cli(ctx):
- """Foo main command.
+def cli(ctx, swh_storage_url, swh_web_url):
+ """Main command for Icinga plugins
"""
+ ctx.ensure_object(dict)
+ ctx.obj['swh_storage_url'] = swh_storage_url
+ ctx.obj['swh_web_url'] = swh_web_url
+
+
+@cli.group(name='check-vault')
+@click.option('--poll-interval', type=int, default=10,
+ help='Interval (in seconds) between two polls to the API, '
+ 'to check for cooking status.')
+@click.pass_context
+def check_vault(ctx, poll_interval):
+ ctx.obj['poll_interval'] = poll_interval
-@cli.command()
-@click.option('--bar', help='Something')
+@check_vault.command(name='directory')
@click.pass_context
-def bar(ctx, bar):
- '''Do something.'''
- click.echo('bar')
+def check_vault_directory(ctx):
+ """Picks a random directory, requests its cooking via swh-web,
+ and waits for completion."""
+ sys.exit(VaultCheck(ctx.obj).main())
diff --git a/swh/icinga_plugins/tests/test_vault.py b/swh/icinga_plugins/tests/test_vault.py
new file mode 100644
--- /dev/null
+++ b/swh/icinga_plugins/tests/test_vault.py
@@ -0,0 +1,229 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import enum
+import json
+import re
+
+from click.testing import CliRunner
+
+from swh.icinga_plugins.cli import cli
+
+
+dir_id = 'ab'*20
+
+response_pending = {
+ "obj_id": dir_id,
+ "obj_type": "directory",
+ "progress_message": "foo",
+ "status": "pending"
+}
+
+response_done = {
+ "fetch_url": f"/api/1/vault/directory/{dir_id}/raw/",
+ "id": 9,
+ "obj_id": dir_id,
+ "obj_type": "directory",
+ "status": "done"
+}
+
+response_failed = {
+ "obj_id": dir_id,
+ "obj_type": "directory",
+ "progress_message": "foobar",
+ "status": "failed"
+}
+
+
+class FakeStorage:
+ def __init__(self, foo, **kwargs):
+ pass
+
+ def directory_get_random(self):
+ return bytes.fromhex(dir_id)
+
+
+def invoke(args, catch_exceptions=False):
+ runner = CliRunner()
+ result = runner.invoke(cli, args)
+ if not catch_exceptions and result.exception:
+ print(result.output)
+ raise result.exception
+ return result
+
+
+def test_vault_immediate_success(requests_mock, mocker):
+
+ class Step(enum.Enum):
+ NOTHING_DONE = 0
+ CHECKED_UNCOOKED = 1
+ REQUESTED_COOKING = 2
+
+ step = Step.NOTHING_DONE
+
+ def post_callback(request, context):
+ nonlocal step
+ if step == Step.CHECKED_UNCOOKED:
+ step = Step.REQUESTED_COOKING
+ return json.dumps(response_pending)
+ else:
+ assert False, step
+
+ def get_callback(request, context):
+ context.json = True
+ nonlocal step
+ if step == Step.NOTHING_DONE:
+ context.status_code = 404
+ step = Step.CHECKED_UNCOOKED
+ elif step == Step.CHECKED_UNCOOKED:
+ assert False
+ elif step == Step.REQUESTED_COOKING:
+ return json.dumps(response_done)
+ else:
+ assert False, step
+
+ requests_mock.get(
+ f'mock://swh-web.example.org/api/1/vault/directory/{dir_id}/',
+ text=get_callback)
+ requests_mock.post(
+ f'mock://swh-web.example.org/api/1/vault/directory/{dir_id}/',
+ text=post_callback)
+
+ get_storage_mock = mocker.patch('swh.icinga_plugins.vault.get_storage')
+ get_storage_mock.side_effect = FakeStorage
+
+ sleep_mock = mocker.patch('time.sleep')
+
+ result = invoke([
+ '--swh-web-url', 'mock://swh-web.example.org',
+ '--swh-storage-url', 'foo://example.org',
+ 'check-vault', 'directory',
+ ])
+
+ assert re.match(
+ rf'VAULT OK - cooking directory {dir_id} took '
+ r'[0-9]\.[0-9]{2}s and succeeded.\n'
+ r"| 'total time' = [0-9]\.[0-9]{2}s",
+ result.output)
+
+ sleep_mock.assert_called_once_with(10)
+
+
+def test_vault_delayed_success(requests_mock, mocker):
+
+ class Step(enum.Enum):
+ NOTHING_DONE = 0
+ CHECKED_UNCOOKED = 1
+ REQUESTED_COOKING = 2
+ PENDING = 3
+
+ step = Step.NOTHING_DONE
+
+ def post_callback(request, context):
+ nonlocal step
+ if step == Step.CHECKED_UNCOOKED:
+ step = Step.REQUESTED_COOKING
+ return json.dumps(response_pending)
+ else:
+ assert False, step
+
+ def get_callback(request, context):
+ context.json = True
+ nonlocal step
+ if step == Step.NOTHING_DONE:
+ context.status_code = 404
+ step = Step.CHECKED_UNCOOKED
+ elif step == Step.CHECKED_UNCOOKED:
+ assert False
+ elif step == Step.REQUESTED_COOKING:
+ step = Step.PENDING
+ return json.dumps(response_pending)
+ elif step == Step.PENDING:
+ return json.dumps(response_done)
+ else:
+ assert False, step
+
+ requests_mock.get(
+ f'mock://swh-web.example.org/api/1/vault/directory/{dir_id}/',
+ text=get_callback)
+ requests_mock.post(
+ f'mock://swh-web.example.org/api/1/vault/directory/{dir_id}/',
+ text=post_callback)
+
+ get_storage_mock = mocker.patch('swh.icinga_plugins.vault.get_storage')
+ get_storage_mock.side_effect = FakeStorage
+
+ sleep_mock = mocker.patch('time.sleep')
+
+ result = invoke([
+ '--swh-web-url', 'mock://swh-web.example.org',
+ '--swh-storage-url', 'foo://example.org',
+ 'check-vault', 'directory',
+ ])
+
+ assert re.match(
+ rf'VAULT OK - cooking directory {dir_id} took '
+ r'[0-9]\.[0-9]{2}s and succeeded.\n'
+ r"| 'total time' = [0-9]\.[0-9]{2}s",
+ result.output)
+
+ assert sleep_mock.call_count == 2
+
+
+def test_vault_failure(requests_mock, mocker):
+
+ class Step(enum.Enum):
+ NOTHING_DONE = 0
+ CHECKED_UNCOOKED = 1
+ REQUESTED_COOKING = 2
+
+ step = Step.NOTHING_DONE
+
+ def post_callback(request, context):
+ nonlocal step
+ if step == Step.CHECKED_UNCOOKED:
+ step = Step.REQUESTED_COOKING
+ return json.dumps(response_pending)
+ else:
+ assert False, step
+
+ def get_callback(request, context):
+ context.json = True
+ nonlocal step
+ if step == Step.NOTHING_DONE:
+ context.status_code = 404
+ step = Step.CHECKED_UNCOOKED
+ elif step == Step.CHECKED_UNCOOKED:
+ assert False
+ elif step == Step.REQUESTED_COOKING:
+ return json.dumps(response_failed)
+ else:
+ assert False, step
+
+ requests_mock.get(
+ f'mock://swh-web.example.org/api/1/vault/directory/{dir_id}/',
+ text=get_callback)
+ requests_mock.post(
+ f'mock://swh-web.example.org/api/1/vault/directory/{dir_id}/',
+ text=post_callback)
+
+ get_storage_mock = mocker.patch('swh.icinga_plugins.vault.get_storage')
+ get_storage_mock.side_effect = FakeStorage
+
+ sleep_mock = mocker.patch('time.sleep')
+
+ result = invoke([
+ '--swh-web-url', 'mock://swh-web.example.org',
+ '--swh-storage-url', 'foo://example.org',
+ 'check-vault', 'directory',
+ ], catch_exceptions=True)
+
+ assert re.match(
+ rf'VAULT CRITICAL - cooking directory {dir_id} took '
+ r'[0-9]\.[0-9]{2}s and failed with: foobar\n'
+ r"| 'total time' = [0-9]\.[0-9]{2}s",
+ result.output)
+
+ sleep_mock.assert_called_once_with(10)
diff --git a/swh/icinga_plugins/vault.py b/swh/icinga_plugins/vault.py
new file mode 100644
--- /dev/null
+++ b/swh/icinga_plugins/vault.py
@@ -0,0 +1,75 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import time
+
+import requests
+
+from swh.storage import get_storage
+
+
+class NoDirectory(Exception):
+ pass
+
+
+class VaultCheck:
+ def __init__(self, obj):
+ self._swh_storage = get_storage('remote', url=obj['swh_storage_url'])
+ self._swh_web_url = obj['swh_web_url']
+ self._poll_interval = obj['poll_interval']
+
+ def _url_for_dir(self, dir_id):
+ return self._swh_web_url + f'/api/1/vault/directory/{dir_id.hex()}/'
+
+ def _pick_directory(self):
+ dir_ = self._swh_storage.directory_get_random()
+ if dir_ is None:
+ raise NoDirectory()
+ return dir_
+
+ def _pick_uncached_directory(self):
+ while True:
+ dir_id = self._pick_directory()
+ response = requests.get(self._url_for_dir(dir_id))
+ if response.status_code == 404:
+ return dir_id
+
+ def main(self):
+ try:
+ dir_id = self._pick_uncached_directory()
+ except NoDirectory:
+ print('VAULT CRITICAL - No directory exists in the archive')
+ return 2
+
+ start_time = time.time()
+ response = requests.post(self._url_for_dir(dir_id))
+ assert response.status_code == 200, (response, response.text)
+ result = response.json()
+ while result['status'] in ('new', 'pending'):
+ time.sleep(self._poll_interval)
+ response = requests.get(self._url_for_dir(dir_id))
+ assert response.status_code == 200, (response, response.text)
+ result = response.json()
+
+ end_time = time.time()
+ total_time = end_time - start_time
+
+ if result['status'] == 'done':
+ print(f'VAULT OK - cooking directory {dir_id.hex()} '
+ f'took {total_time:.2f}s and succeeded.')
+ print(f"| 'total time' = {total_time:.2f}s")
+ return 0
+ elif result['status'] == 'failed':
+ print(f'VAULT CRITICAL - cooking directory {dir_id.hex()} '
+ f'took {total_time:.2f}s and failed with: '
+ f'{result["progress_message"]}')
+ print(f"| 'total time' = {total_time:.2f}s")
+ return 3
+ else:
+ print(f'VAULT CRITICAL - cooking directory {dir_id.hex()} '
+ f'took {total_time:.2f}s and resulted in unknown: '
+ f'status: {result["status"]}')
+ print(f"| 'total time' = {total_time:.2f}s")
+ return 3

File Metadata

Mime Type
text/plain
Expires
Jun 3 2025, 7:36 PM (10 w, 1 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3222302

Event Timeline

About · Developer information · Legal