Changeset View
Changeset View
Standalone View
Standalone View
swh/objstorage/tests/test_objstorage_winery.py
| # Copyright (C) 2021 The Software Heritage developers | # Copyright (C) 2021 The Software Heritage developers | ||||||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||||||
| import os | |||||||||
| import time | import time | ||||||||
| import pytest | import pytest | ||||||||
| import sh | import sh | ||||||||
| from swh.objstorage import exc | from swh.objstorage import exc | ||||||||
| from swh.objstorage.backends.winery.database import DatabaseAdmin | from swh.objstorage.backends.winery.database import DatabaseAdmin | ||||||||
| from swh.objstorage.backends.winery.objstorage import Packer, pack | from swh.objstorage.backends.winery.objstorage import Packer, pack | ||||||||
| from swh.objstorage.backends.winery.stats import Stats | |||||||||
| from swh.objstorage.backends.winery.throttler import ( | from swh.objstorage.backends.winery.throttler import ( | ||||||||
| BandwidthCalculator, | BandwidthCalculator, | ||||||||
| IOThrottler, | IOThrottler, | ||||||||
| LeakyBucket, | LeakyBucket, | ||||||||
| Throttler, | Throttler, | ||||||||
| ) | ) | ||||||||
| from swh.objstorage.factory import get_objstorage | from swh.objstorage.factory import get_objstorage | ||||||||
| ▲ Show 20 Lines • Show All 179 Lines • ▼ Show 20 Lines | |||||||||
| @pytest.mark.asyncio | @pytest.mark.asyncio | ||||||||
| async def test_winery_bench_real(pytestconfig, postgresql, ceph_pool): | async def test_winery_bench_real(pytestconfig, postgresql, ceph_pool): | ||||||||
| dsn = ( | dsn = ( | ||||||||
| f"postgres://{postgresql.info.user}" | f"postgres://{postgresql.info.user}" | ||||||||
| f":@{postgresql.info.host}:{postgresql.info.port}" | f":@{postgresql.info.host}:{postgresql.info.port}" | ||||||||
| ) | ) | ||||||||
| kwargs = { | kwargs = { | ||||||||
| "output_dir": pytestconfig.getoption("--winery-bench-output-directory"), | |||||||||
| "rw_workers": pytestconfig.getoption("--winery-bench-rw-workers"), | "rw_workers": pytestconfig.getoption("--winery-bench-rw-workers"), | ||||||||
| "ro_workers": pytestconfig.getoption("--winery-bench-ro-workers"), | "ro_workers": pytestconfig.getoption("--winery-bench-ro-workers"), | ||||||||
| "shard_max_size": pytestconfig.getoption("--winery-shard-max-size"), | "shard_max_size": pytestconfig.getoption("--winery-shard-max-size"), | ||||||||
| "ro_worker_max_request": pytestconfig.getoption( | "ro_worker_max_request": pytestconfig.getoption( | ||||||||
| "--winery-bench-ro-worker-max-request" | "--winery-bench-ro-worker-max-request" | ||||||||
| ), | ), | ||||||||
| "duration": pytestconfig.getoption("--winery-bench-duration"), | "duration": pytestconfig.getoption("--winery-bench-duration"), | ||||||||
| "base_dsn": dsn, | "base_dsn": dsn, | ||||||||
| "shard_dsn": dsn, | "shard_dsn": dsn, | ||||||||
| "throttle_read": pytestconfig.getoption("--winery-bench-throttle-read"), | "throttle_read": pytestconfig.getoption("--winery-bench-throttle-read"), | ||||||||
| "throttle_write": pytestconfig.getoption("--winery-bench-throttle-write"), | "throttle_write": pytestconfig.getoption("--winery-bench-throttle-write"), | ||||||||
| } | } | ||||||||
| assert await Bench(kwargs).run() == kwargs["rw_workers"] + kwargs["ro_workers"] | count = await Bench(kwargs).run() | ||||||||
| assert count > 0 | |||||||||
| @pytest.mark.asyncio | @pytest.mark.asyncio | ||||||||
| async def test_winery_bench_fake(pytestconfig, mocker): | async def test_winery_bench_fake(pytestconfig, mocker): | ||||||||
| kwargs = { | kwargs = { | ||||||||
| "rw_workers": pytestconfig.getoption("--winery-bench-rw-workers"), | "rw_workers": pytestconfig.getoption("--winery-bench-rw-workers"), | ||||||||
| "ro_workers": pytestconfig.getoption("--winery-bench-ro-workers"), | "ro_workers": pytestconfig.getoption("--winery-bench-ro-workers"), | ||||||||
| "duration": pytestconfig.getoption("--winery-bench-duration"), | "duration": pytestconfig.getoption("--winery-bench-duration"), | ||||||||
| ▲ Show 20 Lines • Show All 137 Lines • ▼ Show 20 Lines | def reader(k): | ||||||||
| return base[k] | return base[k] | ||||||||
| def writer(k, v): | def writer(k, v): | ||||||||
| base[k] = v | base[k] = v | ||||||||
| return True | return True | ||||||||
| assert t.throttle_add(writer, key, content) is True | assert t.throttle_add(writer, key, content) is True | ||||||||
| assert t.throttle_get(reader, key) == content | assert t.throttle_get(reader, key) == content | ||||||||
| def test_winery_stats(tmpdir): | |||||||||
| s = Stats(None) | |||||||||
| assert s.stats_active is False | |||||||||
vlorentzUnsubmitted Done Inline Actions
vlorentz: | |||||||||
Done Inline ActionsCute, thanks for the tip! dachary: Cute, thanks for the tip! | |||||||||
| s = Stats(tmpdir / "stats") | |||||||||
| assert s.stats_active is True | |||||||||
| assert os.path.exists(s.stats_filename) | |||||||||
| size = os.path.getsize(s.stats_filename) | |||||||||
| s._stats_flush_interval = 0 | |||||||||
| k = "KEY" | |||||||||
| v = "CONTENT" | |||||||||
| s.stats_read(k, v) | |||||||||
| s.stats_write(k, v) | |||||||||
| s.stats_read(k, v) | |||||||||
| s.stats_write(k, v) | |||||||||
| s.__del__() | |||||||||
| assert os.path.getsize(s.stats_filename) > size | |||||||||