diff --git a/jobs/swh-mirror.yaml b/jobs/swh-mirror.yaml index fb6ed22..d5f1861 100644 --- a/jobs/swh-mirror.yaml +++ b/jobs/swh-mirror.yaml @@ -1,17 +1,34 @@ - project: name: DFP display-name: swh-mirror jobs: - "{name}" - "{name}/run-tests" + - "{name}/gitlab-run-tests" -- job-template: +- job-template: &swh_mirror_run_tests name: "{name}/run-tests" project-type: pipeline docker_image: tox node: built-in triggers: - timed: "0 4 * * *" dsl: !include-jinja2: templates/swh-mirror-run-tests.groovy.j2 + +- job-template: + name: "{name}/gitlab-run-tests" + gitlab_project: true + properties: + - gitlab: + connection: "{gitlab_connection_name}" + triggers: + - gitlab: + trigger-push: true + trigger-merge-request: true + add-ci-message: true + cancel-pending-builds-on-update: true + # secret jenkins token is generated when executing tox + secret-token: !include-raw: jobs/templates/jenkins-token + <<: *swh_mirror_run_tests diff --git a/jobs/templates/swh-mirror-run-tests.groovy.j2 b/jobs/templates/swh-mirror-run-tests.groovy.j2 index 954fa89..50b35f5 100644 --- a/jobs/templates/swh-mirror-run-tests.groovy.j2 +++ b/jobs/templates/swh-mirror-run-tests.groovy.j2 @@ -1,87 +1,105 @@ pipeline { agent none options { // require "Throttle Concurrent Builds" Jenkins plugin throttleJobProperty( categories: [], limitOneJobWithMatchingParams: false, maxConcurrentPerNode: 1, maxConcurrentTotal: 1, paramsToUseForLimit: '', throttleEnabled: true, throttleOption: 'project', ) } stages { stage('Checkout') { agent { label "built-in" } steps { + {%- if gitlab_project %} + updateGitlabCommitStatus name: 'jenkins', state: 'running' + {%- endif %} + {%- if not gitlab_project %} git url: "https://forge.softwareheritage.org/source/swh-mirror.git" + {%- else %} + git url: "{{gitlab_url}}/swh/infra/swh-mirror.git" + {%- endif %} } } stage('Build Docker test images') { agent { label "built-in" } steps { dir('images') { sh ' ./build_images.sh --write-env-file=image-tag.env' stash name: 'image-tag-env', includes: 'image-tag.env' } } } stage('Run tests') { {% filter indent(width=6) %} {%- include 'templates/includes/agent-docker.groovy.j2' -%} {% endfilter %} environment { SWH_MIRROR_TEST_API_URL = "http://172.17.0.1:5081/api/1" SWH_MIRROR_TEST_KAFKA_BROKER = "broker1.journal.staging.swh.network:9093" SWH_MIRROR_TEST_OBJSTORAGE_HOST = "objstorage.softwareheritage.org" } steps { unstash 'image-tag-env' lock('docker-agent-host-port-5080') { withCredentials([ usernamePassword(credentialsId: 'swh-mirror-test-kafka', usernameVariable: 'SWH_MIRROR_TEST_KAFKA_USERNAME', passwordVariable: 'SWH_MIRROR_TEST_KAFKA_PASSWORD'), usernameColonPassword(credentialsId: 'swh-mirror-test-objstorage', variable: 'SWH_MIRROR_TEST_OBJSTORAGE_CREDENTIALS')]) { sh ''' python3 -m pip install -rrequirements-test.txt SWH_MIRROR_TEST_OBJSTORAGE_URL="https://$SWH_MIRROR_TEST_OBJSTORAGE_CREDENTIALS@$SWH_MIRROR_TEST_OBJSTORAGE_HOST/" \ python3 -m pytest -v --envfile=image-tag.env --timeout=3600 ''' } } } } } post { always { node('built-in') { echo "Clean up Docker stuff" sh ''' docker stack rm swhtest_mirror0 || true # Wait for any remaining containers docker container ls --filter=label=com.docker.stack.namespace=swhtest_mirror0 --quiet | xargs --no-run-if-empty docker container wait || true # Cleanup cruft docker config ls --filter=label=com.docker.stack.namespace=swhtest_mirror0 --quiet | xargs --no-run-if-empty docker config rm || true docker volume ls --filter=label=com.docker.stack.namespace=swhtest_mirror0 --quiet | xargs --no-run-if-empty docker volume rm || true ''' } } + {%- if gitlab_project %} + failure { + updateGitlabCommitStatus name: 'jenkins', state: 'failed' + } + success { + updateGitlabCommitStatus name: 'jenkins', state: 'success' + } + aborted { + updateGitlabCommitStatus name: 'jenkins', state: 'canceled' + } + {%- endif %} } } diff --git a/jobs/tools/setup-gitlab-webhooks.groovy.j2 b/jobs/tools/setup-gitlab-webhooks.groovy.j2 index 3cbdc61..b6644f9 100644 --- a/jobs/tools/setup-gitlab-webhooks.groovy.j2 +++ b/jobs/tools/setup-gitlab-webhooks.groovy.j2 @@ -1,91 +1,93 @@ pipeline { agent any environment { GITLAB_TOKEN = credentials("jenkins-gitlab-token") } stages { stage('Checkout Repository') { steps { checkout([ $class: 'GitSCM', branches: [[name: 'master']], userRemoteConfigs: [[ url: "http://forge.softwareheritage.org/source/swh-jenkins-jobs.git", ]], ]) } } stage('Setup gitlab integration') { steps { script { setupGitlabWebhook("swh/infra/ci-cd/swh-jenkins-jobs", "jenkins-tools/swh-jenkins-jobs-builder", true, true, false) setupGitlabWebhook("swh/infra/ci-cd/swh-jenkins-dockerfiles", "jenkins-tools/gitlab-swh-jenkins-dockerfiles", true, true, false) setupGitlabWebhook("swh/devel/swh-docs", "DDOC/gitlab-builds", true, true, false) + setupGitlabWebhook("swh/infra/swh-mirror", "DFP/gitlab-run-tests", + true, true, false) projects = readYaml(file: 'jobs/swh-packages.yaml') for (project in projects) { if (project.containsKey("project")) { def jenkinsFolder = project.get('project').get('name') def repoName= project.get('project').get('repo_name') def gitlabProjectName = "swh/devel/${repoName}" setupGitlabWebhook(gitlabProjectName, "${jenkinsFolder}/gitlab-tests") setupGitlabWebhook(gitlabProjectName, "${jenkinsFolder}/gitlab-incoming-tag", false, false, true) } } } } } } } void setupGitlabWebhook(gitlabProjectName, jenkinsProjectName, pushEvents = true, mergeRequestEvents = true, tagPushEvents = false) { def webhookUrl = "${jenkins_url}/project/${jenkinsProjectName}" def gitlabProjectEncoded = java.net.URLEncoder.encode(gitlabProjectName, "UTF-8") def payload = """ { "id": "${gitlabProjectEncoded}", "url": "${webhookUrl}", "push_events": "${pushEvents}", "merge_requests_events": "${mergeRequestEvents}", "tag_push_events": "${tagPushEvents}", "token": "{{jenkins_token}}" } """ def url = "${gitlab_url}/api/v4/projects/${gitlabProjectEncoded}/hooks" // get current webhooks of the gitlab project def response = httpRequest httpMode: 'GET', url: url, contentType: 'APPLICATION_JSON', customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]] def hooks = readJSON text: response.content for (hook in hooks) { if (hook.url == webhookUrl) { // update previously set webhook for jenkins job httpRequest httpMode: 'PUT', url: "${url}/${hook.id}", contentType: 'APPLICATION_JSON', requestBody: payload, customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]] return } } // add webhook for jenkins job httpRequest httpMode: 'POST', url: url, contentType: 'APPLICATION_JSON', requestBody: payload, customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]] }