diff --git a/jobs/defaults.yaml b/jobs/defaults.yaml --- a/jobs/defaults.yaml +++ b/jobs/defaults.yaml @@ -1,5 +1,7 @@ - defaults: name: global - gitlab_url: https://gitlab-staging.swh.network + gitlab_host: gitlab-staging.swh.network + gitlab_url: https://{gitlab_host} + gitlab_ssh_url: git@{gitlab_host} gitlab_connection_name: gitlab-staging gitlab_project: false diff --git a/jobs/swh-packages.yaml b/jobs/swh-packages.yaml --- a/jobs/swh-packages.yaml +++ b/jobs/swh-packages.yaml @@ -14,6 +14,9 @@ - "debian/packages/{name}/update-for-release" - "debian/packages/{name}/automatic-backport" - "debian/packages/{name}/gbp-buildpackage" + - "debian/packages/{name}/gitlab-update-for-release" + - "debian/packages/{name}/gitlab-automatic-backport" + - "debian/packages/{name}/gitlab-gbp-buildpackage" - project: name: DAUTH diff --git a/jobs/templates/debian/automatic-backport.groovy.j2 b/jobs/templates/debian/automatic-backport.groovy.j2 --- a/jobs/templates/debian/automatic-backport.groovy.j2 +++ b/jobs/templates/debian/automatic-backport.groovy.j2 @@ -36,7 +36,12 @@ $class: 'GitSCM', branches: [[name: "debian/${params.DESTINATION}"], [name: "refs/tags/${params.GIT_TAG}"]], userRemoteConfigs: [[ + {%- if not gitlab_project %} url: "https://forge.softwareheritage.org/source/${repo_name}.git", + {%- else %} + credentialsId: 'gitlab-public-ci-ssh', + url: '{{gitlab_ssh_url}}:{{gitlab_project_name}}.git', + {%- endif %} ]], ]) } @@ -126,7 +131,11 @@ expression { backport_ok } } steps { + {%- if not gitlab_project %} sshagent (credentials: ['jenkins-public-ci-ssh']) { + {%- else %} + sshagent (credentials: ['gitlab-public-ci-ssh']) { + {%- endif %} script { def git_push = 'git push --follow-tags --all' if (params.DRY_RUN) { diff --git a/jobs/templates/debian/automatic-backport.yaml b/jobs/templates/debian/automatic-backport.yaml --- a/jobs/templates/debian/automatic-backport.yaml +++ b/jobs/templates/debian/automatic-backport.yaml @@ -1,5 +1,5 @@ - job-template: &automatic_backport - name: 'debian/packages/{name}/automatic-backport' + name: "debian/packages/{name}/automatic-backport" description: | Backport Debian package to the given release project-type: pipeline @@ -29,9 +29,15 @@ name: DRY_RUN description: Don't push the resulting changes to a Debian branch default: false - dsl: - !include-jinja2: automatic-backport.groovy.j2 + dsl: !include-jinja2: automatic-backport.groovy.j2 - job-template: - name: 'debian/deps/{name}/automatic-backport' + name: "debian/deps/{name}/automatic-backport" + <<: *automatic_backport + +- job-template: + name: "debian/packages/{name}/gitlab-automatic-backport" + description: | + Backport Debian package to the given release (GitLab) + gitlab_project: true <<: *automatic_backport diff --git a/jobs/templates/debian/gbp-buildpackage.groovy.j2 b/jobs/templates/debian/gbp-buildpackage.groovy.j2 --- a/jobs/templates/debian/gbp-buildpackage.groovy.j2 +++ b/jobs/templates/debian/gbp-buildpackage.groovy.j2 @@ -9,14 +9,21 @@ def upload_target = "${repo_path}/incoming" def repo_command = "umask 002; reprepro -vb ${repo_path} processincoming incoming" +{%- if not gitlab_project %} def backport_job = '/' + (currentBuild.fullProjectName.split('/')[0..-2] + ['automatic-backport']).join('/') +{%- else %} +def backport_job = '/' + (currentBuild.fullProjectName.split('/')[0..-2] + ['gitlab-automatic-backport']).join('/') +{%- endif %} pipeline { agent { label 'debian' } + {%- if not gitlab_project %} environment { PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' } + {%- endif %} stages { + {%- if not gitlab_project %} stage('Notify Phabricator start') { when { beforeAgent true @@ -33,6 +40,7 @@ } } } + {%- endif %} stage('Checkout') { steps { cleanWs() @@ -40,7 +48,11 @@ $class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[ + {%- if not gitlab_project %} url: "https://forge.softwareheritage.org/source/${repo_name}.git", + {%- else %} + url: '{{gitlab_url}}/{{gitlab_project_name}}.git', + {%- endif %} ]], extensions: [ [$class: 'RelativeTargetDirectory', relativeTargetDir: repo_name], @@ -249,6 +261,7 @@ } } } + {%- if not gitlab_project %} post { always { node('swh-tox') { @@ -272,4 +285,5 @@ } } } + {%- endif %} } diff --git a/jobs/templates/debian/gbp-buildpackage.yaml b/jobs/templates/debian/gbp-buildpackage.yaml --- a/jobs/templates/debian/gbp-buildpackage.yaml +++ b/jobs/templates/debian/gbp-buildpackage.yaml @@ -1,8 +1,8 @@ - job-template: &gbp_buildpackage - name: 'debian/packages/{name}/gbp-buildpackage' + name: "debian/packages/{name}/gbp-buildpackage" description: | Build Debian package - auth-token: 'ph4br1cat0r' + auth-token: "ph4br1cat0r" deb-extra-repositories: deb-backport-on-success: unstable-swh>buster-swh project-type: pipeline @@ -22,9 +22,8 @@ description: extra repositories needed to build this package type: checkbox value: !join: - - ',' - - - - incoming + - "," + - - incoming - backports - swh - ceph @@ -32,22 +31,21 @@ - elasticsearch - cassandra value-description: !join: - - ',' - - - - The buildd incoming repository (default for builds from unstable) + - "," + - - The buildd incoming repository (default for builds from unstable) - The debian backports repository (default for builds for stable-swh) - The Software Heritage debian repository (default for builds for swh branches) - The ceph repository - PostgreSQL Global Dvt Group repository (for stable-swh builds requiring latest postgresql) - Elasticsearch 7.x repository (for builds requiring elasticsearch installed) - Cassandra 4.x repository (for builds requiring cassandra installed) - default-value: '{deb-extra-repositories}' + default-value: "{deb-extra-repositories}" - choice: name: BUILD_DEP_RESOLVER description: | the sbuild build dependency resolver (overrides the automatic detection) choices: - - '' + - "" - apt - aptitude - bool: @@ -59,16 +57,20 @@ description: If the build is successful, which backports should we perform? type: checkbox value: !join: - - ',' - - - - unstable-swh>stretch-swh + - "," + - - unstable-swh>stretch-swh - unstable-swh>buster-swh - unstable-swh>bullseye-swh - buster-swh>stretch-swh - default-value: '{deb-backport-on-success}' - dsl: - !include-jinja2: gbp-buildpackage.groovy.j2 + default-value: "{deb-backport-on-success}" + dsl: !include-jinja2: gbp-buildpackage.groovy.j2 - job-template: - name: 'debian/deps/{name}/gbp-buildpackage' + name: "debian/deps/{name}/gbp-buildpackage" + <<: *gbp_buildpackage + +- job-template: + name: "debian/packages/{name}/gitlab-gbp-buildpackage" + description: Build Debian package (GitLab) + gitlab_project: true <<: *gbp_buildpackage diff --git a/jobs/templates/debian/update-for-release.groovy.j2 b/jobs/templates/debian/update-for-release.groovy.j2 --- a/jobs/templates/debian/update-for-release.groovy.j2 +++ b/jobs/templates/debian/update-for-release.groovy.j2 @@ -17,7 +17,12 @@ $class: 'GitSCM', branches: [[name: 'debian/unstable-swh'], [name: 'debian/*'], [name: 'pristine-tar']], userRemoteConfigs: [[ + {%- if not gitlab_project %} url: "https://forge.softwareheritage.org/source/${repo_name}.git", + {%- else %} + credentialsId: 'gitlab-public-ci-ssh', + url: '{{gitlab_ssh_url}}:{{gitlab_project_name}}.git', + {%- endif %} ]], extensions: [ [$class: 'RelativeTargetDirectory', relativeTargetDir: repo_name], @@ -100,7 +105,11 @@ } steps { copyArtifacts( + {%- if not gitlab_project %} projectName: '/{{name}}/pypi-upload', + {%- else %} + projectName: '/{{name}}/gitlab-pypi-upload', + {%- endif %} parameters: 'GIT_TAG=' + params.GIT_TAG, ) withEnv(full_environ) { @@ -133,6 +142,7 @@ } } } + stage('Upload changes') { when { beforeAgent true @@ -140,7 +150,11 @@ } steps { dir (repo_name) { + {%- if gitlab_project %} + sshagent (credentials: ['gitlab-public-ci-ssh']) { + {%- else %} sshagent (credentials: ['jenkins-public-ci-ssh']) { + {%- endif %} script { def git_push = 'git push --follow-tags --all' if (params.DRY_RUN) { diff --git a/jobs/templates/debian/update-for-release.yaml b/jobs/templates/debian/update-for-release.yaml --- a/jobs/templates/debian/update-for-release.yaml +++ b/jobs/templates/debian/update-for-release.yaml @@ -1,5 +1,5 @@ -- job-template: - name: 'debian/packages/{name}/update-for-release' +- job-template: &update_for_release + name: "debian/packages/{name}/update-for-release" project-type: pipeline parameters: - git-parameter: @@ -12,6 +12,9 @@ name: DRY_RUN description: Don't push the resulting changes to a Debian branch default: false - dsl: - !include-jinja2: update-for-release.groovy.j2 + dsl: !include-jinja2: update-for-release.groovy.j2 +- job-template: + name: "debian/packages/{name}/gitlab-update-for-release" + gitlab_project: true + <<: *update_for_release diff --git a/jobs/templates/incoming-tag.groovy.j2 b/jobs/templates/incoming-tag.groovy.j2 --- a/jobs/templates/incoming-tag.groovy.j2 +++ b/jobs/templates/incoming-tag.groovy.j2 @@ -4,6 +4,9 @@ stage('Refresh tag list') { agent any steps { + {%- if gitlab_project %} + updateGitlabCommitStatus name: 'jenkins', state: 'running' + {%- endif %} checkout([ $class: 'GitSCM', {%- if gitlab_project %} @@ -44,7 +47,6 @@ expression { params.GIT_TAG ==~ /v\d+(.\d+)+/ } expression { jobExists('/{{name}}/pypi-upload') } {%- endif %} - } steps { build( @@ -55,48 +57,94 @@ {%- endif %} parameters: [ {%- if gitlab_project %} - string(name: 'GIT_TAG', value: "${env.gitlabSourceBranch}"), + // remove refs/tags/ prefix from source branch + string(name: 'GIT_TAG', value: env.gitlabSourceBranch.substring(10)), {%- else %} string(name: 'GIT_TAG', value: params.GIT_TAG), {%- endif %} string(name: 'PYPI_HOST', value: '{{incoming_tag_auto_pypi_host}}'), ], + wait: true, ) } } - {%- if not gitlab_project %} + stage('Update Debian packaging for new release') { + when { + {%- if not gitlab_project %} expression { params.GIT_TAG ==~ /v\d+(.\d+)+/ } expression { jobExists('/debian/packages/{{name}}/update-for-release') } + {%- else %} + expression { "${env.gitlabSourceBranch}" ==~ /refs\/tags\/v\d+(.\d+)+/ } + expression { jobExists('/debian/packages/{{name}}/gitlab-update-for-release') } + {%- endif %} } + steps { build( + {%- if not gitlab_project %} job: '/debian/packages/{{name}}/update-for-release', + {%- else %} + job: '/debian/packages/{{name}}/gitlab-update-for-release', + {%- endif %} parameters: [ + {%- if not gitlab_project %} string(name: 'GIT_TAG', value: params.GIT_TAG), + {%- else %} + // remove refs/tags/ prefix from source branch + string(name: 'GIT_TAG', value: env.gitlabSourceBranch.substring(10)), + {%- endif %} ], - wait: false, + wait: true, ) } } + stage('Build Debian package') { when { + {%- if not gitlab_project %} expression { params.GIT_TAG ==~ /debian\/.*/ } expression { !(params.GIT_TAG ==~ /debian\/upstream\/.*/) } expression { jobExists('/debian/packages/{{name}}/gbp-buildpackage') } + {%- else %} + expression { "${env.gitlabSourceBranch}" ==~ /refs\/tags\/debian\/.*/ } + expression { !("${env.gitlabSourceBranch}" ==~ /refs\/tags\/debian\/upstream\/.*/) } + expression { jobExists('/debian/packages/{{name}}/gitlab-gbp-buildpackage') } + {%- endif %} } steps { build( + {%- if not gitlab_project %} job: '/debian/packages/{{name}}/gbp-buildpackage', parameters: [ string(name: 'GIT_REVISION', value: params.GIT_TAG), booleanParam(name: 'DO_UPLOAD', value: true), ], - wait: false, + {%- else %} + job: '/debian/packages/{{name}}/gitlab-gbp-buildpackage', + parameters: [ + // remove refs/tags/ prefix from source branch + string(name: 'GIT_REVISION', value: env.gitlabSourceBranch.substring(10)), + booleanParam(name: 'DO_UPLOAD', value: true), + ], + {%- endif %} + wait: true, ) } } - {%- endif %} } + {%- if gitlab_project %} + post { + failure { + updateGitlabCommitStatus name: 'jenkins', state: 'failed' + } + success { + updateGitlabCommitStatus name: 'jenkins', state: 'success' + } + aborted { + updateGitlabCommitStatus name: 'jenkins', state: 'canceled' + } + } + {%- endif %} } diff --git a/jobs/templates/incoming-tag.yaml b/jobs/templates/incoming-tag.yaml --- a/jobs/templates/incoming-tag.yaml +++ b/jobs/templates/incoming-tag.yaml @@ -27,6 +27,9 @@ display-name: Incoming tag (GitLab) gitlab_project: true parameters: + properties: + - gitlab: + connection: "{gitlab_connection_name}" triggers: - gitlab: trigger-push: true diff --git a/jobs/templates/swh-pipeline.yaml b/jobs/templates/swh-pipeline.yaml --- a/jobs/templates/swh-pipeline.yaml +++ b/jobs/templates/swh-pipeline.yaml @@ -50,6 +50,8 @@ cancel-pending-builds-on-update: true # secret jenkins token is generated when executing tox secret-token: !include-raw: jenkins-token + branch-filter-type: RegexBasedFilter + target-branch-regex: ^(?!(debian|pristine-tar)).* gitlab_project: true parameters: - string: diff --git a/jobs/tools/jenkins-jobs-builder.groovy.j2 b/jobs/tools/jenkins-jobs-builder.groovy.j2 --- a/jobs/tools/jenkins-jobs-builder.groovy.j2 +++ b/jobs/tools/jenkins-jobs-builder.groovy.j2 @@ -28,7 +28,6 @@ when { // really update jenkins jobs configuration when pushing commits to master // or when manually triggering the job from Jenkins UI - expression { !env.gitlabMergeRequestIid } anyOf { expression { !env.gitlabSourceBranch } expression { "${env.gitlabSourceBranch}" == "master" }