diff --git a/jobs/dependency-packages.yaml b/jobs/dependency-packages.yaml index 4ef0f59..a53b4e5 100644 --- a/jobs/dependency-packages.yaml +++ b/jobs/dependency-packages.yaml @@ -1,159 +1,163 @@ - job-group: - name: 'dependency-jobs-{name}' + name: "dependency-jobs-{name}" + gitlab_project_name: "swh/infra/ci-cd/debs/{repo_name}" jobs: - "debian/deps/{name}" - "debian/deps/{name}/incoming-tag" - "debian/deps/{name}/gbp-buildpackage" - "debian/deps/{name}/automatic-backport" + - "debian/deps/{name}/gitlab-incoming-tag" + - "debian/deps/{name}/gitlab-gbp-buildpackage" + - "debian/deps/{name}/gitlab-automatic-backport" - project: name: PPDW display-name: python3-dulwich repo_name: python3-dulwich pkg: dulwich jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PTDTM display-name: python3-django-test-migrations repo_name: python3-django-test-migrations pkg: django-test-migrations jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PMK display-name: python3-mirakuru repo_name: python3-mirakuru pkg: mirakuru jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PPL display-name: python3-plotille repo_name: python3-plotille pkg: plotille jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PTPP display-name: python3-pytest-postgresql repo_name: python3-pytest-postgresql pkg: pytest-postgresql jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PPF display-name: python3-port-for repo_name: python3-port-for pkg: port-for jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PSD display-name: python3-sqlitedict repo_name: python3-sqlitedict pkg: sqlitedict jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PDJR display-name: python3-django-js-reverse repo_name: python3-django-js-reverse pkg: django-js-reverse jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PJOS display-name: python3-jose repo_name: python3-jose pkg: python3-jose jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PRMD display-name: python3-repomd repo_name: python3-repomd pkg: repomd jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PKCL display-name: python3-keycloak repo_name: python3-keycloak pkg: python3-keycloak jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PATS display-name: python3-attr-strict repo_name: python3-attr-strict pkg: python3-attr-strict jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PPTRE display-name: python3-pytest-redis repo_name: python3-pytest-redis pkg: pytest_redis jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PCASS display-name: python-cassandra-driver repo_name: python-cassandra-driver pkg: python-cassandra-driver jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PPSASL display-name: python-pure-sasl repo_name: python-pure-sasl pkg: python-pure-sasl jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PDWL display-name: django-webpack-loader repo_name: django-webpack-loader pkg: python3-django-webpack-loader jobs: - "dependency-jobs-{name}" - project: name: PPPE display-name: python3-prometheus-pve-exporter repo_name: python3-prometheus-pve-exporter pkg: prometheus-pve-exporter jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" - project: name: PSTATIO display-name: python3-statusio repo_name: python3-statusio pkg: python3-statusio jobs: - "dependency-jobs-{name}" - project: name: PTS display-name: python3-tree-sitter repo_name: python3-tree-sitter pkg: tree-sitter jobs: - - 'dependency-jobs-{name}' + - "dependency-jobs-{name}" diff --git a/jobs/templates/debian/automatic-backport.yaml b/jobs/templates/debian/automatic-backport.yaml index 2974760..fd443a2 100644 --- a/jobs/templates/debian/automatic-backport.yaml +++ b/jobs/templates/debian/automatic-backport.yaml @@ -1,43 +1,48 @@ - job-template: &automatic_backport name: "debian/packages/{name}/automatic-backport" description: | Backport Debian package to the given release project-type: pipeline parameters: - string: name: GIT_TAG description: git tag to backport - choice: name: SOURCE description: check that the source branch corresponds to this before issuing the backport choices: - unstable-swh - unstable - bookworm-swh - bullseye-swh - buster-swh default: unstable-swh - choice: name: DESTINATION description: the destination branch the backport targets choices: - buster-swh - bullseye-swh - bookworm-swh default: buster-swh - bool: name: DRY_RUN description: Don't push the resulting changes to a Debian branch default: false dsl: !include-jinja2: automatic-backport.groovy.j2 +- job-template: + name: "debian/packages/{name}/gitlab-automatic-backport" + description: | + Backport Debian package to the given release (GitLab) + gitlab_project: true + <<: *automatic_backport + - job-template: name: "debian/deps/{name}/automatic-backport" <<: *automatic_backport - job-template: - name: "debian/packages/{name}/gitlab-automatic-backport" - description: | - Backport Debian package to the given release (GitLab) + name: "debian/deps/{name}/gitlab-automatic-backport" gitlab_project: true <<: *automatic_backport diff --git a/jobs/templates/debian/gbp-buildpackage.yaml b/jobs/templates/debian/gbp-buildpackage.yaml index f4879f3..5f9949c 100644 --- a/jobs/templates/debian/gbp-buildpackage.yaml +++ b/jobs/templates/debian/gbp-buildpackage.yaml @@ -1,76 +1,81 @@ - job-template: &gbp_buildpackage name: "debian/packages/{name}/gbp-buildpackage" description: | Build Debian package auth-token: "ph4br1cat0r" deb-extra-repositories: deb-backport-on-success: unstable-swh>buster-swh project-type: pipeline properties: - build-discarder: num-to-keep: 10 parameters: - string: name: GIT_REVISION default: debian/unstable-swh description: Git revision to build - string: name: PHID description: PHID of the phabricator target object on which results will be reported - extended-choice: name: EXTRA_REPOSITORIES description: extra repositories needed to build this package type: checkbox value: !join: - "," - - incoming - backports - swh - ceph - pgdg - elasticsearch - cassandra value-description: !join: - "," - - The buildd incoming repository (default for builds from unstable) - The debian backports repository (default for builds for stable-swh) - The Software Heritage debian repository (default for builds for swh branches) - The ceph repository - PostgreSQL Global Dvt Group repository (for stable-swh builds requiring latest postgresql) - Elasticsearch 7.x repository (for builds requiring elasticsearch installed) - Cassandra 4.x repository (for builds requiring cassandra installed) default-value: "{deb-extra-repositories}" - choice: name: BUILD_DEP_RESOLVER description: | the sbuild build dependency resolver (overrides the automatic detection) choices: - "" - apt - aptitude - bool: name: DO_UPLOAD description: upload the built package default: false - extended-choice: name: BACKPORT_ON_SUCCESS description: If the build is successful, which backports should we perform? type: checkbox value: !join: - "," - - unstable-swh>stretch-swh - unstable-swh>buster-swh - unstable-swh>bullseye-swh - buster-swh>stretch-swh default-value: "{deb-backport-on-success}" dsl: !include-jinja2: gbp-buildpackage.groovy.j2 +- job-template: + name: "debian/packages/{name}/gitlab-gbp-buildpackage" + description: Build Debian package (GitLab) + gitlab_project: true + <<: *gbp_buildpackage + - job-template: name: "debian/deps/{name}/gbp-buildpackage" <<: *gbp_buildpackage - job-template: - name: "debian/packages/{name}/gitlab-gbp-buildpackage" - description: Build Debian package (GitLab) + name: "debian/deps/{name}/gitlab-gbp-buildpackage" gitlab_project: true <<: *gbp_buildpackage diff --git a/jobs/templates/dependency-incoming-tag.groovy.j2 b/jobs/templates/dependency-incoming-tag.groovy.j2 index 1f379f6..8ddc388 100644 --- a/jobs/templates/dependency-incoming-tag.groovy.j2 +++ b/jobs/templates/dependency-incoming-tag.groovy.j2 @@ -1,41 +1,87 @@ pipeline { agent none stages { stage('Refresh tag list') { agent any steps { + {%- if gitlab_project %} + updateGitlabCommitStatus name: 'jenkins', state: 'running' + {%- endif %} checkout([ $class: 'GitSCM', + {% if not gitlab_project %} userRemoteConfigs: [[ url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git', ]], branches: [[ name: params.GIT_TAG, ]], browser: [ $class: 'Phabricator', repo: '{{repo_name}}', repoUrl: 'https://forge.softwareheritage.org/', ], + {%- else %} + userRemoteConfigs: [[ + name:'origin', url: '{{gitlab_url}}/{{gitlab_project_name}}.git', + refspec: '+refs/tags/*:refs/remotes/origin/tags*' + ]], + branches: [[ + name: "${env.gitlabSourceBranch}" + ]], + browser: [ + $class: 'GitLab', + repoUrl: '{{gitlab_url}}/{{gitlab_project_name}}' + ], + extensions: [[$class: 'CloneOption', honorRefspec: true]], + {%- endif %} ]) } } stage('Build Debian package') { when { + {%- if not gitlab_project %} expression { params.GIT_TAG ==~ /debian\/.*/ } expression { !(params.GIT_TAG ==~ /debian\/upstream\/.*/) } expression { jobExists('/debian/deps/{{name}}/gbp-buildpackage') } + {%- else %} + expression { "${env.gitlabSourceBranch}" ==~ /refs\/tags\/debian\/.*/ } + expression { !("${env.gitlabSourceBranch}" ==~ /refs\/tags\/debian\/upstream\/.*/) } + expression { jobExists('/debian/deps/{{name}}/gitlab-gbp-buildpackage') } + {%- endif %} } steps { build( + {%- if not gitlab_project %} job: '/debian/deps/{{name}}/gbp-buildpackage', parameters: [ string(name: 'GIT_REVISION', value: params.GIT_TAG), booleanParam(name: 'DO_UPLOAD', value: true), ], - wait: false, + {%- else %} + job: '/debian/deps/{{name}}/gitlab-gbp-buildpackage', + parameters: [ + // remove refs/tags/ prefix from source branch + string(name: 'GIT_REVISION', value: env.gitlabSourceBranch.substring(10)), + booleanParam(name: 'DO_UPLOAD', value: false), + ], + {%- endif %} + wait: true, ) } } } + {%- if gitlab_project %} + post { + failure { + updateGitlabCommitStatus name: 'jenkins', state: 'failed' + } + success { + updateGitlabCommitStatus name: 'jenkins', state: 'success' + } + aborted { + updateGitlabCommitStatus name: 'jenkins', state: 'canceled' + } + } + {%- endif %} } diff --git a/jobs/templates/incoming-tag.yaml b/jobs/templates/incoming-tag.yaml index 2b1b89c..e1367e6 100644 --- a/jobs/templates/incoming-tag.yaml +++ b/jobs/templates/incoming-tag.yaml @@ -1,40 +1,57 @@ - job-template: &incoming_tag name: "{name}/incoming-tag" display-name: Incoming tag project-type: pipeline auth-token: "ph4br1cat0r" incoming_tag_auto_pypi_host: pypi.org sandbox: true properties: - build-discarder: num-to-keep: 20 parameters: - git-parameter: name: GIT_TAG description: git tag to process type: PT_TAG sortMode: DESCENDING_SMART selectedValue: TOP dsl: !include-jinja2: incoming-tag.groovy.j2 - job-template: + name: "{name}/gitlab-incoming-tag" + display-name: Incoming tag (GitLab) + gitlab_project: true + parameters: + properties: + - gitlab: + connection: "{gitlab_connection_name}" + triggers: + - gitlab: + trigger-push: true + trigger-merge-request: false + add-ci-message: true + # secret jenkins token is generated when executing tox + secret-token: !include-raw: jenkins-token + <<: *incoming_tag + +- job-template: &dep_incoming_tag name: "debian/deps/{name}/incoming-tag" dsl: !include-jinja2: dependency-incoming-tag.groovy.j2 <<: *incoming_tag - job-template: - name: "{name}/gitlab-incoming-tag" + name: "debian/deps/{name}/gitlab-incoming-tag" display-name: Incoming tag (GitLab) gitlab_project: true parameters: properties: - gitlab: connection: "{gitlab_connection_name}" triggers: - gitlab: trigger-push: true trigger-merge-request: false add-ci-message: true # secret jenkins token is generated when executing tox secret-token: !include-raw: jenkins-token - <<: *incoming_tag + <<: *dep_incoming_tag diff --git a/jobs/tools/setup-gitlab-webhooks.groovy.j2 b/jobs/tools/setup-gitlab-webhooks.groovy.j2 index b6644f9..944378d 100644 --- a/jobs/tools/setup-gitlab-webhooks.groovy.j2 +++ b/jobs/tools/setup-gitlab-webhooks.groovy.j2 @@ -1,93 +1,104 @@ pipeline { agent any environment { GITLAB_TOKEN = credentials("jenkins-gitlab-token") } stages { stage('Checkout Repository') { steps { checkout([ $class: 'GitSCM', branches: [[name: 'master']], userRemoteConfigs: [[ url: "http://forge.softwareheritage.org/source/swh-jenkins-jobs.git", ]], ]) } } stage('Setup gitlab integration') { steps { script { setupGitlabWebhook("swh/infra/ci-cd/swh-jenkins-jobs", "jenkins-tools/swh-jenkins-jobs-builder", true, true, false) setupGitlabWebhook("swh/infra/ci-cd/swh-jenkins-dockerfiles", "jenkins-tools/gitlab-swh-jenkins-dockerfiles", true, true, false) setupGitlabWebhook("swh/devel/swh-docs", "DDOC/gitlab-builds", true, true, false) setupGitlabWebhook("swh/infra/swh-mirror", "DFP/gitlab-run-tests", true, true, false) projects = readYaml(file: 'jobs/swh-packages.yaml') for (project in projects) { if (project.containsKey("project")) { def jenkinsFolder = project.get('project').get('name') def repoName= project.get('project').get('repo_name') def gitlabProjectName = "swh/devel/${repoName}" setupGitlabWebhook(gitlabProjectName, "${jenkinsFolder}/gitlab-tests") setupGitlabWebhook(gitlabProjectName, "${jenkinsFolder}/gitlab-incoming-tag", false, false, true) } } + + projects = readYaml(file: 'jobs/dependency-packages.yaml') + for (project in projects) { + if (project.containsKey("project")) { + def jenkinsFolder = project.get('project').get('name') + jenkinsFolder = "debian/deps/${jenkinsFolder}/gitlab-incoming-tag" + def repoName= project.get('project').get('repo_name') + def gitlabProjectName = "swh/infra/ci-cd/debs/${repoName}" + setupGitlabWebhook(gitlabProjectName, jenkinsFolder, false, false, true) + } + } } } } } } void setupGitlabWebhook(gitlabProjectName, jenkinsProjectName, pushEvents = true, mergeRequestEvents = true, tagPushEvents = false) { def webhookUrl = "${jenkins_url}/project/${jenkinsProjectName}" def gitlabProjectEncoded = java.net.URLEncoder.encode(gitlabProjectName, "UTF-8") def payload = """ { "id": "${gitlabProjectEncoded}", "url": "${webhookUrl}", "push_events": "${pushEvents}", "merge_requests_events": "${mergeRequestEvents}", "tag_push_events": "${tagPushEvents}", "token": "{{jenkins_token}}" } """ def url = "${gitlab_url}/api/v4/projects/${gitlabProjectEncoded}/hooks" // get current webhooks of the gitlab project def response = httpRequest httpMode: 'GET', url: url, contentType: 'APPLICATION_JSON', customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]] def hooks = readJSON text: response.content for (hook in hooks) { if (hook.url == webhookUrl) { // update previously set webhook for jenkins job httpRequest httpMode: 'PUT', url: "${url}/${hook.id}", contentType: 'APPLICATION_JSON', requestBody: payload, customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]] return } } // add webhook for jenkins job httpRequest httpMode: 'POST', url: url, contentType: 'APPLICATION_JSON', requestBody: payload, customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]] }