diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cc45b3..23fe219 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,49 +1,50 @@ +exclude: winery-test-environment/mitogen-strategy repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.4.0 hooks: - id: trailing-whitespace - id: check-json - id: check-yaml - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.3 hooks: - id: flake8 - repo: https://github.com/codespell-project/codespell rev: v1.16.0 hooks: - id: codespell - repo: local hooks: - id: mypy name: mypy entry: mypy args: [swh] pass_filenames: false language: system types: [python] - repo: https://github.com/PyCQA/isort rev: 5.5.2 hooks: - id: isort - repo: https://github.com/python/black rev: 19.10b0 hooks: - id: black # unfortunately, we are far from being able to enable this... # - repo: https://github.com/PyCQA/pydocstyle.git # rev: 4.0.0 # hooks: # - id: pydocstyle # name: pydocstyle # description: pydocstyle is a static analysis tool for checking compliance with Python docstring conventions. # entry: pydocstyle --convention=google # language: python # types: [python] diff --git a/setup.cfg b/setup.cfg index 8d79b7e..7bf85d1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,7 @@ [flake8] # E203: whitespaces before ':' # E231: missing whitespace after ',' # W503: line break before binary operator ignore = E203,E231,W503 max-line-length = 88 +extend-exclude = winery-test-environment/context,winery-test-environment/mitogen-strategy diff --git a/tox.ini b/tox.ini index 5ac7b32..8e06ba8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,78 +1,79 @@ [tox] envlist=flake8,py3,winery,mypy [testenv] extras = testing deps = pytest-cov dev: pdbpp commands = pytest --cov={envsitepackagesdir}/swh/objstorage \ {envsitepackagesdir}/swh/objstorage \ --cov-branch --cov-report term-missing {posargs} [testenv:winery] allowlist_externals = bash +passenv = * commands = bash {toxinidir}/winery-test-environment/remote-tox.sh {posargs} [testenv:black] skip_install = true deps = black==19.10b0 commands = {envpython} -m black --check swh [testenv:flake8] skip_install = true deps = flake8 commands = {envpython} -m flake8 [testenv:mypy] extras = testing deps = mypy==0.920 commands = mypy swh # build documentation outside swh-environment using the current # git HEAD of swh-docs, is executed on CI for each diff to prevent # breaking doc build [testenv:sphinx] whitelist_externals = make usedevelop = true extras = testing deps = # fetch and install swh-docs in develop mode -e git+https://forge.softwareheritage.org/source/swh-docs#egg=swh.docs setenv = SWH_PACKAGE_DOC_TOX_BUILD = 1 # turn warnings into errors SPHINXOPTS = -W commands = make -I ../.tox/sphinx/src/swh-docs/swh/ -C docs # build documentation only inside swh-environment using local state # of swh-docs package [testenv:sphinx-dev] whitelist_externals = make usedevelop = true extras = testing deps = # install swh-docs in develop mode -e ../swh-docs setenv = SWH_PACKAGE_DOC_TOX_BUILD = 1 # turn warnings into errors SPHINXOPTS = -W commands = make -I ../.tox/sphinx-dev/src/swh-docs/swh/ -C docs diff --git a/winery-test-environment/ansible.cfg b/winery-test-environment/ansible.cfg index 4aa7c8b..23d33d2 100644 --- a/winery-test-environment/ansible.cfg +++ b/winery-test-environment/ansible.cfg @@ -1,7 +1,9 @@ [defaults] +strategy_plugins = mitogen-strategy +strategy = mitogen_linear private_key_file = ./context/cluster_key host_key_checking = false [ssh_connection] ssh_args = -F context/ssh-config scp_if_ssh = True diff --git a/winery-test-environment/build-vms.sh b/winery-test-environment/build-vms.sh index 7ba37e3..737559a 100755 --- a/winery-test-environment/build-vms.sh +++ b/winery-test-environment/build-vms.sh @@ -1,134 +1,135 @@ #!/bin/bash # Copyright (C) 2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information set -e : ${LIBVIRT_URI:=qemu:///system} VIRSH="virsh --connect $LIBVIRT_URI" VIRT_INSTALL="virt-install --connect $LIBVIRT_URI" function ssh_key() { if ! test -f cluster_key; then ssh-keygen -f cluster_key -N '' -t rsa fi + chmod 600 cluster_key } function stop() { local ids="$@" for id in $ids ; do $VIRSH destroy ceph$id >& /dev/null || true $VIRSH undefine ceph$id >& /dev/null || true rm -f ceph$id.qcow2 rm -f disk$id*.img done $VIRSH net-destroy ceph >& /dev/null || true $VIRSH net-undefine ceph >& /dev/null || true } function start() { local ids="$@" ssh_key > ssh-config if ! test -f debian-11.qcow2 ; then sudo virt-builder debian-11 --output debian-11.qcow2 --size 10G --format qcow2 --install sudo --run-command 'dpkg-reconfigure --frontend=noninteractive openssh-server' --run-command 'useradd -s /bin/bash -m debian || true ; echo "debian ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-debian' --ssh-inject debian:file:cluster_key.pub --edit '/etc/network/interfaces: s/ens2/enp1s0/' fi if ! $VIRSH net-list --name | grep ceph ; then cat > ceph-net.xml < ceph EOF $VIRSH net-define ceph-net.xml $VIRSH net-start ceph fi for id in $ids ; do $VIRSH destroy ceph$id >& /dev/null || true $VIRSH undefine ceph$id >& /dev/null || true rm -f ceph$id.qcow2 cp --sparse=always debian-11.qcow2 ceph$id.qcow2 sudo virt-sysprep -a ceph$id.qcow2 --enable customize --hostname ceph$id $VIRT_INSTALL --network network=ceph,mac=52:54:00:00:00:0$id --boot hd --name ceph$id --memory 2048 --vcpus 1 --cpu host --disk path=$(pwd)/ceph$id.qcow2,bus=virtio,format=qcow2 --os-type=linux --os-variant=debian10 --graphics none --noautoconsole case $id in 1) ;; 2) $VIRSH detach-device ceph$id ../rng.xml --live for drive in b c ; do # # Without the sleep it fails with: # # error: Failed to attach disk # error: internal error: No more available PCI slots # sleep 10 rm -f disk$id$drive.img qemu-img create -f raw disk$id$drive.img 20G sudo chown libvirt-qemu disk$id$drive.img $VIRSH attach-disk ceph$id --source $(pwd)/disk$id$drive.img --target vd$drive --persistent done ;; *) rm -f disk$id.img qemu-img create -f raw disk$id.img 20G sudo chown libvirt-qemu disk$id.img $VIRSH attach-disk ceph$id --source $(pwd)/disk$id.img --target vdb --persistent ;; esac cat >> ssh-config <