diff --git a/jobs/swh-mirror.yaml b/jobs/swh-mirror.yaml new file mode 100644 --- /dev/null +++ b/jobs/swh-mirror.yaml @@ -0,0 +1,23 @@ +- project: + name: DFP + display-name: swh-mirror + jobs: + - "{name}" + - "{name}/run-tests" + +- job-template: + name: "{name}/run-tests" + project-type: pipeline + docker_image: tox + node: built-in + + scm: + - git: + url: https://forge.softwareheritage.org/source/swh-mirror.git + branches: + - "*/master" + + triggers: + - timed: "@midnight" + + dsl: !include-jinja2: templates/swh-mirror-run-tests.groovy.j2 diff --git a/jobs/templates/swh-mirror-run-tests.groovy.j2 b/jobs/templates/swh-mirror-run-tests.groovy.j2 new file mode 100644 --- /dev/null +++ b/jobs/templates/swh-mirror-run-tests.groovy.j2 @@ -0,0 +1,93 @@ +pipeline { + agent none + + options { + // require "Throttle Concurrent Builds" Jenkins plugin + throttleJobProperty( + categories: [], + limitOneJobWithMatchingParams: false, + maxConcurrentPerNode: 1, + maxConcurrentTotal: 1, + paramsToUseForLimit: '', + throttleEnabled: true, + throttleOption: 'project', + ) + } + + stages { + stage('Checkout') { + agent { + label "built-in" + } + + steps { + git url: "https://forge.softwareheritage.org/source/swh-mirror.git" + } + } + + stage('Build Docker test images') { + agent { + label "built-in" + } + + steps { + dir('images') { + sh ' ./build_images.sh --write-env-file=image-tag.env' + stash name: 'image-tag-env', includes: 'image-tag.env' + } + } + } + + stage('Run tests') { + // FIXME: Including 'includes/agent-docker.groovy.j2' here makes + // jenkins-job-builder fail with “TypeError: expected string or bytes-like + // object”. It looks related to variables used by the included template but + // it’s totally unclear why it cannot find them. + agent { + docker { + reuseNode true + image 'swh-jenkins/tox' + args '--tmpfs /tmp:exec -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker -u jenkins:docker --mount type=volume,src=shared-jenkins-cachedir,dst=/home/jenkins/.cache' + } + } + + environment { + SWH_MIRROR_TEST_API_URL = "http://172.17.0.1:5081/api/1" + SWH_MIRROR_TEST_KAFKA_BROKER = "broker1.journal.staging.swh.network:9093" + SWH_MIRROR_TEST_OBJSTORAGE_HOST = "objstorage.softwareheritage.org" + } + + steps { + unstash 'image-tag-env' + withCredentials([ + usernamePassword(credentialsId: 'swh-mirror-test-kafka', + usernameVariable: 'SWH_MIRROR_TEST_KAFKA_USERNAME', + passwordVariable: 'SWH_MIRROR_TEST_KAFKA_PASSWORD'), + usernameColonPassword(credentialsId: 'swh-mirror-test-objstorage', + variable: 'SWH_MIRROR_TEST_OBJSTORAGE_CREDENTIALS')]) { + sh ''' + python3 -m pip install -rrequirements-test.txt + SWH_MIRROR_TEST_OBJSTORAGE_URL="https://$SWH_MIRROR_TEST_OBJSTORAGE_CREDENTIALS@$SWH_MIRROR_TEST_OBJSTORAGE_HOST/" \ + python3 -m pytest -v --envfile=image-tag.env --timeout=3600 + ''' + } + } + } + } + + post { + always { + node('built-in') { + echo "Clean up Docker stuff" + sh ''' + docker stack rm swhtest_mirror0 || true + # Wait for any remaining containers + docker container ls --filter=label=com.docker.stack.namespace=swhtest_mirror0 --quiet | xargs --no-run-if-empty docker container wait || true + # Cleanup cruft + docker config ls --filter=label=com.docker.stack.namespace=swhtest_mirror0 --quiet | xargs --no-run-if-empty docker config rm || true + docker volume ls --filter=label=com.docker.stack.namespace=swhtest_mirror0 --quiet | xargs --no-run-if-empty docker volume rm || true + ''' + } + } + } +}