diff --git a/jobs/templates/swh-docker-dev.groovy.j2 b/jobs/templates/swh-docker-dev.groovy.j2 index e16d703..d35043f 100644 --- a/jobs/templates/swh-docker-dev.groovy.j2 +++ b/jobs/templates/swh-docker-dev.groovy.j2 @@ -1,55 +1,57 @@ pipeline { agent none options { timeout(time: 30, unit: 'MINUTES') timestamps() } stages { stage('Checkout') { agent { label "built-in" } steps { git url: "https://forge.softwareheritage.org/source/swh-environment.git" } } stage('Build Docker images') { agent { label "built-in" } steps { dir('docker') { sh 'docker build --pull --no-cache -t swh/stack .' } } } stage('Run tests') { {% filter indent(width=6) %} {%- include 'templates/includes/agent-docker.groovy.j2' -%} {% endfilter %} steps { - dir('docker') { - sh 'python3 -m tox -- -v' + lock('docker-agent-host-port-5080') { + dir('docker') { + sh 'python3 -m tox -- -v' + } } } } } post { always { node('built-in') { archiveArtifacts( allowEmptyArchive: true, artifacts: 'docker/**/*.logs', ) } } } } diff --git a/jobs/templates/swh-mirror-run-tests.groovy.j2 b/jobs/templates/swh-mirror-run-tests.groovy.j2 index bcd3334..954fa89 100644 --- a/jobs/templates/swh-mirror-run-tests.groovy.j2 +++ b/jobs/templates/swh-mirror-run-tests.groovy.j2 @@ -1,85 +1,87 @@ pipeline { agent none options { // require "Throttle Concurrent Builds" Jenkins plugin throttleJobProperty( categories: [], limitOneJobWithMatchingParams: false, maxConcurrentPerNode: 1, maxConcurrentTotal: 1, paramsToUseForLimit: '', throttleEnabled: true, throttleOption: 'project', ) } stages { stage('Checkout') { agent { label "built-in" } steps { git url: "https://forge.softwareheritage.org/source/swh-mirror.git" } } stage('Build Docker test images') { agent { label "built-in" } steps { dir('images') { sh ' ./build_images.sh --write-env-file=image-tag.env' stash name: 'image-tag-env', includes: 'image-tag.env' } } } stage('Run tests') { {% filter indent(width=6) %} {%- include 'templates/includes/agent-docker.groovy.j2' -%} {% endfilter %} environment { SWH_MIRROR_TEST_API_URL = "http://172.17.0.1:5081/api/1" SWH_MIRROR_TEST_KAFKA_BROKER = "broker1.journal.staging.swh.network:9093" SWH_MIRROR_TEST_OBJSTORAGE_HOST = "objstorage.softwareheritage.org" } steps { unstash 'image-tag-env' - withCredentials([ - usernamePassword(credentialsId: 'swh-mirror-test-kafka', - usernameVariable: 'SWH_MIRROR_TEST_KAFKA_USERNAME', - passwordVariable: 'SWH_MIRROR_TEST_KAFKA_PASSWORD'), - usernameColonPassword(credentialsId: 'swh-mirror-test-objstorage', - variable: 'SWH_MIRROR_TEST_OBJSTORAGE_CREDENTIALS')]) { - sh ''' - python3 -m pip install -rrequirements-test.txt - SWH_MIRROR_TEST_OBJSTORAGE_URL="https://$SWH_MIRROR_TEST_OBJSTORAGE_CREDENTIALS@$SWH_MIRROR_TEST_OBJSTORAGE_HOST/" \ - python3 -m pytest -v --envfile=image-tag.env --timeout=3600 - ''' + lock('docker-agent-host-port-5080') { + withCredentials([ + usernamePassword(credentialsId: 'swh-mirror-test-kafka', + usernameVariable: 'SWH_MIRROR_TEST_KAFKA_USERNAME', + passwordVariable: 'SWH_MIRROR_TEST_KAFKA_PASSWORD'), + usernameColonPassword(credentialsId: 'swh-mirror-test-objstorage', + variable: 'SWH_MIRROR_TEST_OBJSTORAGE_CREDENTIALS')]) { + sh ''' + python3 -m pip install -rrequirements-test.txt + SWH_MIRROR_TEST_OBJSTORAGE_URL="https://$SWH_MIRROR_TEST_OBJSTORAGE_CREDENTIALS@$SWH_MIRROR_TEST_OBJSTORAGE_HOST/" \ + python3 -m pytest -v --envfile=image-tag.env --timeout=3600 + ''' + } } } } } post { always { node('built-in') { echo "Clean up Docker stuff" sh ''' docker stack rm swhtest_mirror0 || true # Wait for any remaining containers docker container ls --filter=label=com.docker.stack.namespace=swhtest_mirror0 --quiet | xargs --no-run-if-empty docker container wait || true # Cleanup cruft docker config ls --filter=label=com.docker.stack.namespace=swhtest_mirror0 --quiet | xargs --no-run-if-empty docker config rm || true docker volume ls --filter=label=com.docker.stack.namespace=swhtest_mirror0 --quiet | xargs --no-run-if-empty docker volume rm || true ''' } } } }