diff --git a/jobs/defaults.yaml b/jobs/defaults.yaml index cb1e056..06fa34a 100644 --- a/jobs/defaults.yaml +++ b/jobs/defaults.yaml @@ -1,5 +1,7 @@ - defaults: name: global - gitlab_url: https://gitlab-staging.swh.network + gitlab_host: gitlab-staging.swh.network + gitlab_url: https://{gitlab_host} + gitlab_ssh_url: git@{gitlab_host} gitlab_connection_name: gitlab-staging gitlab_project: false diff --git a/jobs/swh-packages.yaml b/jobs/swh-packages.yaml index 5ee4775..2ee0f29 100644 --- a/jobs/swh-packages.yaml +++ b/jobs/swh-packages.yaml @@ -1,349 +1,352 @@ - job-group: name: "swh-jobs-{name}" gitlab_project_name: "swh/devel/{repo_name}" jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tests-on-diff" - "{name}/gitlab-tests" - "{name}/gitlab-incoming-tag" - "{name}/gitlab-pypi-upload" - "debian/packages/{name}" - "debian/packages/{name}/update-for-release" - "debian/packages/{name}/automatic-backport" - "debian/packages/{name}/gbp-buildpackage" + - "debian/packages/{name}/gitlab-update-for-release" + - "debian/packages/{name}/gitlab-automatic-backport" + - "debian/packages/{name}/gitlab-gbp-buildpackage" - project: name: DAUTH display-name: swh-auth repo_name: swh-auth pkg: auth python_module: swh.auth jobs: - "swh-jobs-{name}" - project: name: DCORE display-name: swh-core repo_name: swh-core pkg: core python_module: swh.core jobs: - "swh-jobs-{name}" - project: name: DCNT display-name: swh-counters repo_name: swh-counters pkg: counters python_module: swh.counters jobs: - "swh-jobs-{name}" - project: name: DDATASET display-name: swh-dataset repo_name: swh-dataset pkg: dataset python_module: swh.dataset jobs: - "swh-jobs-{name}" - project: name: DDEP display-name: swh-deposit repo_name: swh-deposit pkg: deposit python_module: swh.deposit jobs: - "swh-jobs-{name}" - project: name: DGRPH display-name: swh-graph repo_name: swh-graph pkg: graph python_module: swh.graph jobs: - "swh-jobs-{name}" - project: name: DGQL display-name: swh-graphql repo_name: swh-graphql pkg: graphql python_module: swh.graphql jobs: - "swh-jobs-{name}" - project: name: DCIDX display-name: swh-indexer repo_name: swh-indexer pkg: indexer python_module: swh.indexer jobs: - "swh-jobs-{name}" - project: name: DICP display-name: swh-icinga-plugins repo_name: swh-icinga-plugins pkg: icinga-plugins python_module: swh.icinga_plugins jobs: - "swh-jobs-{name}" - project: name: DJNL display-name: swh-journal repo_name: swh-journal docker_image: kafka pkg: journal python_module: swh.journal jobs: - "swh-jobs-{name}" - project: name: DLS display-name: swh-lister repo_name: swh-lister pkg: lister python_module: swh.lister jobs: - "swh-jobs-{name}" - project: name: DLDBZR display-name: swh-loader-bzr repo_name: swh-loader-bzr pkg: loader.bzr python_module: swh.loader.bzr jobs: - "swh-jobs-{name}" - project: name: DLDBASE display-name: swh-loader-core repo_name: swh-loader-core pkg: loader.core python_module: swh.loader.core jobs: - "swh-jobs-{name}" - project: name: DLDCVS display-name: swh-loader-cvs repo_name: swh-loader-cvs pkg: loader.cvs python_module: swh.loader.cvs include_bdist: false jobs: - "swh-jobs-{name}" - project: name: DLDG display-name: swh-loader-git repo_name: swh-loader-git pkg: loader.git python_module: swh.loader.git jobs: - "swh-jobs-{name}" - project: name: DLDMD display-name: swh-loader-metadata repo_name: swh-loader-metadata pkg: loader.metadata python_module: swh.loader.metadata jobs: - "swh-jobs-{name}" - project: name: DLDHG display-name: swh-loader-mercurial repo_name: swh-loader-mercurial pkg: loader.mercurial python_module: swh.loader.mercurial jobs: - "swh-jobs-{name}" - project: name: DLDSVN display-name: swh-loader-svn repo_name: swh-loader-svn pkg: loader.svn python_module: swh.loader.svn jobs: - "swh-jobs-{name}" - project: name: DMOD display-name: swh-model repo_name: swh-model pkg: model python_module: swh.model jobs: - "swh-jobs-{name}" - project: name: DMFCD display-name: swh-clearlydefined repo_name: swh-clearlydefined pkg: clearlydefined python_module: swh.clearlydefined jobs: - "swh-jobs-{name}" - project: name: DOBJS display-name: swh-objstorage repo_name: swh-objstorage pkg: objstorage deb-extra-repositories: ceph python_module: swh.objstorage jobs: - "swh-jobs-{name}" - project: name: DOBJSRPL display-name: swh-objstorage-replayer repo_name: swh-objstorage-replayer pkg: objstorage.replayer python_module: swh.objstorage.replayer jobs: - "swh-jobs-{name}" - project: name: DOPH display-name: swh-perfecthash repo_name: swh-perfecthash pkg: perfecthash python_module: swh.perfecthash include_bdist: false jobs: - "swh-jobs-{name}" - project: name: DPROV display-name: swh-provenance repo_name: swh-provenance pkg: provenance python_module: swh.provenance timeout: 15 jobs: - "swh-jobs-{name}" - project: name: DSEA display-name: swh-search repo_name: swh-search pkg: search deb-extra-repositories: elasticsearch python_module: swh.search jobs: - "swh-jobs-{name}" - project: name: TSBX display-name: swh-sandbox repo_name: swh-sandbox pkg: sandbox incoming_tag_auto_pypi_host: test.pypi.org python_module: swh.sandbox jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tests-on-diff" - project: name: DTSCN display-name: swh-scanner repo_name: swh-scanner pkg: scanner python_module: swh.scanner jobs: - "swh-jobs-{name}" - project: name: DSCH display-name: swh-scheduler repo_name: swh-scheduler pkg: scheduler python_module: swh.scheduler jobs: - "swh-jobs-{name}" - project: name: DSCRUB display-name: swh-scrubber repo_name: swh-scrubber pkg: scrubber python_module: swh.scrubber jobs: - "swh-jobs-{name}" - project: name: DSTO display-name: swh-storage repo_name: swh-storage pkg: storage deb-extra-repositories: cassandra python_module: swh.storage timeout: 25 jobs: - "swh-jobs-{name}" - project: name: DVAU display-name: swh-vault repo_name: swh-vault pkg: vault python_module: swh.vault jobs: - "swh-jobs-{name}" - project: name: DWAPPS display-name: swh-web repo_name: swh-web pkg: web python_module: swh.web do_cypress: true nb_cypress_runners: 4 timeout: 30 max_concurrent: 3 jobs: - "swh-jobs-{name}" - project: name: DWCLI display-name: swh-web-client repo_name: swh-web-client pkg: web-client python_module: swh.web.client jobs: - "swh-jobs-{name}" - project: name: DFUSE display-name: swh-fuse repo_name: swh-fuse pkg: fuse python_module: swh.fuse docker_options: --privileged --device /dev/fuse jobs: - "swh-jobs-{name}" - project: name: DTPL display-name: swh-py-template repo_name: swh-py-template pkg: py-template python_module: swh.foo jobs: - "swh-jobs-{name}" diff --git a/jobs/templates/debian/automatic-backport.groovy.j2 b/jobs/templates/debian/automatic-backport.groovy.j2 index ee06edc..5ea06f8 100644 --- a/jobs/templates/debian/automatic-backport.groovy.j2 +++ b/jobs/templates/debian/automatic-backport.groovy.j2 @@ -1,142 +1,151 @@ def repo_name = '{{repo_name}}' def environment def release switch (params.DESTINATION) { case ~/^stretch(-.*|)$/: release = 9 break case ~/^buster(-.*|)$/: release = 10 break case ~/^bullseye(-.*|)$/: release = 11 break case ~/^bookworm(-.*|)$/: release = 12 break case ~/^trixie(-.*|)$/: release = 13 break default: release = 'UNKNOWN' break } def backport_ok = false pipeline { agent { label 'debian' } stages { stage('Checkout') { steps { cleanWs() checkout([ $class: 'GitSCM', branches: [[name: "debian/${params.DESTINATION}"], [name: "refs/tags/${params.GIT_TAG}"]], userRemoteConfigs: [[ + {%- if not gitlab_project %} url: "https://forge.softwareheritage.org/source/${repo_name}.git", + {%- else %} + credentialsId: 'gitlab-public-ci-ssh', + url: '{{gitlab_ssh_url}}:{{gitlab_project_name}}.git', + {%- endif %} ]], ]) } } stage('Check the source branch') { steps { script { def current_release = sh( script: "dpkg-parsechangelog -SDistribution", returnStdout: true, ).trim(); backport_ok = current_release == params.SOURCE && release != 'UNKNOWN' } } } stage('Seed the environment') { when { beforeAgent true expression { backport_ok } } steps { script { def hostname = sh( script: "hostname --fqdn", returnStdout: true, ).trim(); def short_hostname = hostname - '.internal.softwareheritage.org'; environment = [ "DEBEMAIL=jenkins@${hostname}", "DEBFULLNAME=Software Heritage autobuilder (on ${short_hostname})", ] } } } stage('Merge package version to destination branch') { when { beforeAgent true expression { backport_ok } } steps { withEnv(environment) { sh """ git checkout debian/${params.DESTINATION} || git checkout -b debian/${params.DESTINATION} ${params.GIT_TAG} git merge ${params.GIT_TAG} --no-commit --no-edit || true git checkout ${params.GIT_TAG} -- debian/changelog debian/gbp.conf sed -i 's!^debian-branch\\s*=.*\$!debian-branch=debian/${params.DESTINATION}!' debian/gbp.conf git add debian/changelog debian/gbp.conf git commit --no-verify -m "Merge tag '${params.GIT_TAG}' into debian/${params.DESTINATION}" git show """ } } } stage('Update changelog for backport') { when { beforeAgent true expression { backport_ok } } steps { withEnv(environment) { sh """ dch -l ~bpo${release}+ -D ${params.DESTINATION} --force-distribution 'Rebuild for ${params.DESTINATION}' git add debian/changelog git commit --no-verify -m "Updated backport on ${params.DESTINATION} from ${params.GIT_TAG} (${params.SOURCE})" git show """ } } } stage('Tag package') { when { beforeAgent true expression { backport_ok } } steps { withEnv(environment) { sh 'gbp buildpackage --git-tag-only' } } } stage('Upload changes') { when { beforeAgent true expression { backport_ok } } steps { + {%- if not gitlab_project %} sshagent (credentials: ['jenkins-public-ci-ssh']) { + {%- else %} + sshagent (credentials: ['gitlab-public-ci-ssh']) { + {%- endif %} script { def git_push = 'git push --follow-tags --all' if (params.DRY_RUN) { git_push += ' -n' } sh(script: git_push) } } } } } } diff --git a/jobs/templates/debian/automatic-backport.yaml b/jobs/templates/debian/automatic-backport.yaml index b7d5f8f..2974760 100644 --- a/jobs/templates/debian/automatic-backport.yaml +++ b/jobs/templates/debian/automatic-backport.yaml @@ -1,37 +1,43 @@ - job-template: &automatic_backport - name: 'debian/packages/{name}/automatic-backport' + name: "debian/packages/{name}/automatic-backport" description: | Backport Debian package to the given release project-type: pipeline parameters: - string: name: GIT_TAG description: git tag to backport - choice: name: SOURCE description: check that the source branch corresponds to this before issuing the backport choices: - unstable-swh - unstable - bookworm-swh - bullseye-swh - buster-swh default: unstable-swh - choice: name: DESTINATION description: the destination branch the backport targets choices: - buster-swh - bullseye-swh - bookworm-swh default: buster-swh - bool: name: DRY_RUN description: Don't push the resulting changes to a Debian branch default: false - dsl: - !include-jinja2: automatic-backport.groovy.j2 + dsl: !include-jinja2: automatic-backport.groovy.j2 - job-template: - name: 'debian/deps/{name}/automatic-backport' + name: "debian/deps/{name}/automatic-backport" + <<: *automatic_backport + +- job-template: + name: "debian/packages/{name}/gitlab-automatic-backport" + description: | + Backport Debian package to the given release (GitLab) + gitlab_project: true <<: *automatic_backport diff --git a/jobs/templates/debian/gbp-buildpackage.groovy.j2 b/jobs/templates/debian/gbp-buildpackage.groovy.j2 index 9480416..a09f1d7 100644 --- a/jobs/templates/debian/gbp-buildpackage.groovy.j2 +++ b/jobs/templates/debian/gbp-buildpackage.groovy.j2 @@ -1,275 +1,289 @@ def repo_name = '{{repo_name}}' def changelog_distribution def repo_host = 'pergamon.internal.softwareheritage.org' def repo_user = 'swhdebianrepo' def repo_path = '/srv/softwareheritage/repository' def upload_target = "${repo_path}/incoming" def repo_command = "umask 002; reprepro -vb ${repo_path} processincoming incoming" +{%- if not gitlab_project %} def backport_job = '/' + (currentBuild.fullProjectName.split('/')[0..-2] + ['automatic-backport']).join('/') +{%- else %} +def backport_job = '/' + (currentBuild.fullProjectName.split('/')[0..-2] + ['gitlab-automatic-backport']).join('/') +{%- endif %} pipeline { agent { label 'debian' } + {%- if not gitlab_project %} environment { PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' } + {%- endif %} stages { + {%- if not gitlab_project %} stage('Notify Phabricator start') { when { beforeAgent true expression { params.PHID } } agent { label 'swh-tox' } steps { withCredentials([ string(credentialsId: 'swh-public-ci', variable: 'PHAB_CONDUIT_TOKEN')]) { sh ''' python3 -m pyarcanist send-message work $PHID ''' } } } + {%- endif %} stage('Checkout') { steps { cleanWs() checkout([ $class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[ + {%- if not gitlab_project %} url: "https://forge.softwareheritage.org/source/${repo_name}.git", + {%- else %} + url: '{{gitlab_url}}/{{gitlab_project_name}}.git', + {%- endif %} ]], extensions: [ [$class: 'RelativeTargetDirectory', relativeTargetDir: repo_name], ], ]) script { dir(repo_name) { if(!fileExists('debian/changelog')) { error('Trying to build a debian package without a debian/changelog') } changelog_distribution = sh( script: 'dpkg-parsechangelog -SDistribution', returnStdout: true, ).trim() def parsed_gbp_config = readProperties( file: 'debian/gbp.conf', defaults: ['debian-branch': 'master'], ) def debian_branch = parsed_gbp_config['debian-branch'] def pristine_tar = parsed_gbp_config['pristine-tar'] def upstream_branch = parsed_gbp_config['upstream-branch'] sh """ git checkout -b ${debian_branch} [ ${pristine_tar} = "True" ] && git branch -f pristine-tar origin/pristine-tar git branch -f ${upstream_branch} origin/${upstream_branch} """ } } } } stage('Build package') { when { beforeAgent true expression { changelog_distribution != 'UNRELEASED' } } steps { script { dir (repo_name) { def wanted_extra_repositories = params.EXTRA_REPOSITORIES.split(',') def build_dep_resolver def extra_repositories = [] def extra_repository_keys = [] def base_distribution = changelog_distribution.split('-')[0] def backports = changelog_distribution.endsWith('-backports') def swh = changelog_distribution.endsWith('-swh') if (base_distribution in ['unstable', 'experimental'] || 'incoming' in wanted_extra_repositories) { def suites = [] if (base_distribution == 'unstable') { suites = ['buildd-unstable'] } else if (base_distribution == 'experimental') { suites = ['buildd-unstable', 'buildd-experimental'] } else { suites = ["buildd-${base_distribution}-proposed-updates"] if (backports || swh) { suites.add("buildd-${base_distribution}-backports") } } suites.each {suite -> extra_repositories.add("deb http://incoming.debian.org/debian-buildd/ ${suite} main") } } if (swh || 'swh' in wanted_extra_repositories) { def swh_distribution = "${base_distribution}-swh" if (base_distribution in ['unstable', 'experimental']) { swh_distribution = 'unstable' } extra_repositories.add("deb [trusted=yes] https://debian.softwareheritage.org/ ${swh_distribution} main") } if ((backports || swh || 'backports' in wanted_extra_repositories) && !(base_distribution in ['unstable', 'experimental'])) { extra_repositories.add("deb http://deb.debian.org/debian/ ${base_distribution}-backports main") build_dep_resolver = 'aptitude' } if ('ceph' in wanted_extra_repositories && !(base_distribution in ['unstable', 'experimental'])) { if (base_distribution != 'stretch') { extra_repositories.add("deb http://download.proxmox.com/debian/ceph-octopus/ ${base_distribution} main") extra_repository_keys.add('proxmox.asc') } else { extra_repositories.add("deb http://download.proxmox.com/debian/ceph-luminous/ ${base_distribution} main") extra_repository_keys.add('proxmox.asc') } } if ('pgdg' in wanted_extra_repositories && !(base_distribution in ['unstable', 'experimental'])) { extra_repositories.add("deb http://apt.postgresql.org/pub/repos/apt/ ${base_distribution}-pgdg main") extra_repository_keys.add('postgres.asc') } if ('elasticsearch' in wanted_extra_repositories) { extra_repositories.add("deb https://artifacts.elastic.co/packages/7.x/apt stable main") extra_repository_keys.add('elasticsearch.asc') } if ('cassandra' in wanted_extra_repositories) { extra_repositories.add("deb https://debian.cassandra.apache.org 40x main") extra_repository_keys.add('cassandra.asc') } if (params.BUILD_DEP_RESOLVER) { build_dep_resolver = params.BUILD_DEP_RESOLVER } def hostname = sh( script: "hostname --fqdn", returnStdout: true, ).trim(); def short_hostname = hostname - '.internal.softwareheritage.org'; def uploader = "Software Heritage autobuilder (on ${short_hostname}) " def gbp_buildpackage = [ 'gbp buildpackage', '--git-builder=sbuild', '--nolog', '--batch', '--no-clean-source', '--no-run-lintian', '--arch-all', '--source', '--force-orig-source', "--uploader='${uploader}'", ] if (build_dep_resolver != null) { gbp_buildpackage.add("--build-dep-resolver=${build_dep_resolver}") } extra_repositories.each { repo -> gbp_buildpackage.add("--extra-repository='${repo}'") } extra_repository_keys.each { key -> gbp_buildpackage.add("--extra-repository-key=/usr/share/keyrings/extra-repositories/${key}") } def gbp_buildpackage_cmd = gbp_buildpackage.join(' ') sh(script: gbp_buildpackage_cmd) } if (params.DO_UPLOAD) { sh(script: 'debsign *.changes') } archiveArtifacts( artifacts: sh( script: 'dcmd echo *.changes', returnStdout: true ).split().join(','), fingerprint: true, ) } } } stage('Upload package') { when { beforeAgent true expression { changelog_distribution != 'UNRELEASED' } expression { params.DO_UPLOAD } } steps { sshagent (credentials: ['jenkins-debian-repo-ssh']) { sh """ dcmd rsync -v *.changes ${repo_user}@${repo_host}:${upload_target} ssh ${repo_user}@${repo_host} '${repo_command}' """ } } } stage('Prepare backport') { when { beforeAgent true expression { changelog_distribution != 'UNRELEASED' } expression { params.BACKPORT_ON_SUCCESS } expression { jobExists(backport_job) } } steps { script { params.BACKPORT_ON_SUCCESS.split(',').each { bpo_pair -> def (src_suite, dst_suite) = bpo_pair.split('>') if (src_suite == changelog_distribution) { build( job: backport_job, parameters: [ string(name: 'GIT_TAG', value: params.GIT_REVISION), string(name: 'SOURCE', value: src_suite), string(name: 'DESTINATION', value: dst_suite), ], wait: false, ) } } } } } } + {%- if not gitlab_project %} post { always { node('swh-tox') { withCredentials([ string(credentialsId: 'swh-public-ci', variable: 'PHAB_CONDUIT_TOKEN')]) { withEnv(["JOBSTATUS=${currentBuild.currentResult}"]) { sh ''' if [ "$JOBSTATUS" = "SUCCESS" ]; then MSGTYPE=pass else MSGTYPE=fail fi echo "Current job status is $JOBSTATUS -> $MGSTYPE" if [ -n "$PHID" ]; then python3 -m pyarcanist send-message $MSGTYPE $PHID fi ''' } } } } } + {%- endif %} } diff --git a/jobs/templates/debian/gbp-buildpackage.yaml b/jobs/templates/debian/gbp-buildpackage.yaml index 6928b64..f4879f3 100644 --- a/jobs/templates/debian/gbp-buildpackage.yaml +++ b/jobs/templates/debian/gbp-buildpackage.yaml @@ -1,74 +1,76 @@ - job-template: &gbp_buildpackage - name: 'debian/packages/{name}/gbp-buildpackage' + name: "debian/packages/{name}/gbp-buildpackage" description: | Build Debian package - auth-token: 'ph4br1cat0r' + auth-token: "ph4br1cat0r" deb-extra-repositories: deb-backport-on-success: unstable-swh>buster-swh project-type: pipeline properties: - build-discarder: num-to-keep: 10 parameters: - string: name: GIT_REVISION default: debian/unstable-swh description: Git revision to build - string: name: PHID description: PHID of the phabricator target object on which results will be reported - extended-choice: name: EXTRA_REPOSITORIES description: extra repositories needed to build this package type: checkbox value: !join: - - ',' - - - - incoming + - "," + - - incoming - backports - swh - ceph - pgdg - elasticsearch - cassandra value-description: !join: - - ',' - - - - The buildd incoming repository (default for builds from unstable) + - "," + - - The buildd incoming repository (default for builds from unstable) - The debian backports repository (default for builds for stable-swh) - The Software Heritage debian repository (default for builds for swh branches) - The ceph repository - PostgreSQL Global Dvt Group repository (for stable-swh builds requiring latest postgresql) - Elasticsearch 7.x repository (for builds requiring elasticsearch installed) - Cassandra 4.x repository (for builds requiring cassandra installed) - default-value: '{deb-extra-repositories}' + default-value: "{deb-extra-repositories}" - choice: name: BUILD_DEP_RESOLVER description: | the sbuild build dependency resolver (overrides the automatic detection) choices: - - '' + - "" - apt - aptitude - bool: name: DO_UPLOAD description: upload the built package default: false - extended-choice: name: BACKPORT_ON_SUCCESS description: If the build is successful, which backports should we perform? type: checkbox value: !join: - - ',' - - - - unstable-swh>stretch-swh + - "," + - - unstable-swh>stretch-swh - unstable-swh>buster-swh - unstable-swh>bullseye-swh - buster-swh>stretch-swh - default-value: '{deb-backport-on-success}' - dsl: - !include-jinja2: gbp-buildpackage.groovy.j2 + default-value: "{deb-backport-on-success}" + dsl: !include-jinja2: gbp-buildpackage.groovy.j2 - job-template: - name: 'debian/deps/{name}/gbp-buildpackage' + name: "debian/deps/{name}/gbp-buildpackage" + <<: *gbp_buildpackage + +- job-template: + name: "debian/packages/{name}/gitlab-gbp-buildpackage" + description: Build Debian package (GitLab) + gitlab_project: true <<: *gbp_buildpackage diff --git a/jobs/templates/debian/update-for-release.groovy.j2 b/jobs/templates/debian/update-for-release.groovy.j2 index 7d2782c..1736087 100644 --- a/jobs/templates/debian/update-for-release.groovy.j2 +++ b/jobs/templates/debian/update-for-release.groovy.j2 @@ -1,157 +1,171 @@ def upstream_tag = params.GIT_TAG def version = upstream_tag.substring(1) def repo_name = '{{repo_name}}' def python_module = '{{python_module}}' def full_environ def debian_upstream_tag_exists = false pipeline { agent { label 'debian' } stages { stage('Checkout') { steps { cleanWs() checkout([ $class: 'GitSCM', branches: [[name: 'debian/unstable-swh'], [name: 'debian/*'], [name: 'pristine-tar']], userRemoteConfigs: [[ + {%- if not gitlab_project %} url: "https://forge.softwareheritage.org/source/${repo_name}.git", + {%- else %} + credentialsId: 'gitlab-public-ci-ssh', + url: '{{gitlab_ssh_url}}:{{gitlab_project_name}}.git', + {%- endif %} ]], extensions: [ [$class: 'RelativeTargetDirectory', relativeTargetDir: repo_name], ], poll: false, ]) dir (repo_name) { sh ''' git checkout -B pristine-tar origin/pristine-tar git checkout -B debian/upstream origin/debian/upstream git checkout -B debian/unstable-swh origin/debian/unstable-swh ''' script { def tag_exists_retval = sh( script: "git rev-parse --verify debian/upstream/${version}", returnStatus: true ) debian_upstream_tag_exists = (tag_exists_retval == 0) } } } } stage('Get author information from tag') { when { beforeAgent true expression { !debian_upstream_tag_exists } } steps { dir (repo_name) { // This script retrieves the author data for the tag that we're currently processing script { // Check if the tag is annotated or not def objecttype = sh( script: "git tag -l --format='%(*objecttype)' ${upstream_tag}", returnStdout: true, ).trim(); def tagger = !objecttype.isEmpty() ? 'tagger' : 'author'; def tagger_name = sh( script: "git tag -l --format='%(${tagger}name)' ${upstream_tag}", returnStdout: true, ).trim(); def tagger_email = sh( script: "git tag -l --format='%(${tagger}email)' ${upstream_tag}", returnStdout: true, ).trim(); tagger_email = tagger_email.substring(1, tagger_email.length() - 1); def tagger_date = sh( script: "git tag -l --format='%(${tagger}date:iso)' ${upstream_tag}", returnStdout: true, ).trim(); def hostname = sh( script: "hostname --fqdn", returnStdout: true, ).trim(); def short_hostname = hostname - '.internal.softwareheritage.org'; full_environ = [ "DEBEMAIL=jenkins@${hostname}", "DEBFULLNAME=Software Heritage autobuilder (on ${short_hostname})", "UPSTREAM_TAG=${upstream_tag}", "UPSTREAM_TAGGER_NAME=${tagger_name}", "UPSTREAM_TAGGER_EMAIL=${tagger_email}", "UPSTREAM_TAGGER_DATE=${tagger_date}", "UPSTREAM_VERSION=${version}", "PYTHON_MODULE=${python_module}", ]; } } } } stage('gbp import-orig') { when { beforeAgent true expression { !debian_upstream_tag_exists } } steps { copyArtifacts( + {%- if not gitlab_project %} projectName: '/{{name}}/pypi-upload', + {%- else %} + projectName: '/{{name}}/gitlab-pypi-upload', + {%- endif %} parameters: 'GIT_TAG=' + params.GIT_TAG, ) withEnv(full_environ) { dir (repo_name) { sh ''' gbp import-orig --merge-mode=replace --no-symlink-orig -u ${UPSTREAM_VERSION} ../dist/${PYTHON_MODULE}-${UPSTREAM_VERSION}.tar.gz ''' } } } } stage('unstable-swh changelog') { when { beforeAgent true expression { !debian_upstream_tag_exists } } steps { withEnv(full_environ) { dir(repo_name) { sh ''' dch -v ${UPSTREAM_VERSION}-1~swh1 "New upstream release ${UPSTREAM_VERSION}\n - (tagged by ${UPSTREAM_TAGGER_NAME} <${UPSTREAM_TAGGER_EMAIL}> on ${UPSTREAM_TAGGER_DATE})" dch "Upstream changes:\n$(git tag -l --format='%(contents:subject)%0a%(contents:body)' ${UPSTREAM_TAG} | sed -E -e '/^$/d' -e 's/^ *(- *)?//' -e 's/^/ - /')" dch -D unstable-swh --force-distribution '' git add debian/changelog git commit --no-verify -m "Updated debian changelog for version ${UPSTREAM_VERSION}" git show gbp buildpackage --git-tag-only ''' } } } } + stage('Upload changes') { when { beforeAgent true expression { !debian_upstream_tag_exists } } steps { dir (repo_name) { + {%- if gitlab_project %} + sshagent (credentials: ['gitlab-public-ci-ssh']) { + {%- else %} sshagent (credentials: ['jenkins-public-ci-ssh']) { + {%- endif %} script { def git_push = 'git push --follow-tags --all' if (params.DRY_RUN) { git_push += ' -n' } sh(script: git_push) } } } } } } } diff --git a/jobs/templates/debian/update-for-release.yaml b/jobs/templates/debian/update-for-release.yaml index 2c84cc1..3322f2d 100644 --- a/jobs/templates/debian/update-for-release.yaml +++ b/jobs/templates/debian/update-for-release.yaml @@ -1,17 +1,20 @@ -- job-template: - name: 'debian/packages/{name}/update-for-release' +- job-template: &update_for_release + name: "debian/packages/{name}/update-for-release" project-type: pipeline parameters: - git-parameter: name: GIT_TAG type: PT_TAG description: git tag to process sortMode: DESCENDING_SMART selectedValue: TOP - bool: name: DRY_RUN description: Don't push the resulting changes to a Debian branch default: false - dsl: - !include-jinja2: update-for-release.groovy.j2 + dsl: !include-jinja2: update-for-release.groovy.j2 +- job-template: + name: "debian/packages/{name}/gitlab-update-for-release" + gitlab_project: true + <<: *update_for_release diff --git a/jobs/templates/incoming-tag.groovy.j2 b/jobs/templates/incoming-tag.groovy.j2 index 6fee916..8a014a4 100644 --- a/jobs/templates/incoming-tag.groovy.j2 +++ b/jobs/templates/incoming-tag.groovy.j2 @@ -1,102 +1,150 @@ pipeline { agent none stages { stage('Refresh tag list') { agent any steps { + {%- if gitlab_project %} + updateGitlabCommitStatus name: 'jenkins', state: 'running' + {%- endif %} checkout([ $class: 'GitSCM', {%- if gitlab_project %} userRemoteConfigs: [[ name:'origin', url: '{{gitlab_url}}/{{gitlab_project_name}}.git', refspec: '+refs/tags/*:refs/remotes/origin/tags*' ]], branches: [[ name: "${env.gitlabSourceBranch}" ]], browser: [ $class: 'GitLab', repoUrl: '{{gitlab_url}}/{{gitlab_project_name}}' ], extensions: [[$class: 'CloneOption', honorRefspec: true]], {%- else %} userRemoteConfigs: [[ url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git', ]], branches: [[ name: params.GIT_TAG, ]], browser: [ $class: 'Phabricator', repo: '{{repo_name}}', repoUrl: 'https://forge.softwareheritage.org/', ], {%- endif %} ]) } } stage('Build and upload PyPI package') { when { {%- if gitlab_project %} expression { "${env.gitlabSourceBranch}" ==~ /refs\/tags\/v\d+(.\d+)+/ } expression { jobExists('/{{name}}/gitlab-pypi-upload') } {%- else %} expression { params.GIT_TAG ==~ /v\d+(.\d+)+/ } expression { jobExists('/{{name}}/pypi-upload') } {%- endif %} - } steps { build( {%- if gitlab_project %} job: '/{{name}}/gitlab-pypi-upload', {%- else %} job: '/{{name}}/pypi-upload', {%- endif %} parameters: [ {%- if gitlab_project %} - string(name: 'GIT_TAG', value: "${env.gitlabSourceBranch}"), + // remove refs/tags/ prefix from source branch + string(name: 'GIT_TAG', value: env.gitlabSourceBranch.substring(10)), {%- else %} string(name: 'GIT_TAG', value: params.GIT_TAG), {%- endif %} string(name: 'PYPI_HOST', value: '{{incoming_tag_auto_pypi_host}}'), ], + wait: true, ) } } - {%- if not gitlab_project %} + stage('Update Debian packaging for new release') { + when { + {%- if not gitlab_project %} expression { params.GIT_TAG ==~ /v\d+(.\d+)+/ } expression { jobExists('/debian/packages/{{name}}/update-for-release') } + {%- else %} + expression { "${env.gitlabSourceBranch}" ==~ /refs\/tags\/v\d+(.\d+)+/ } + expression { jobExists('/debian/packages/{{name}}/gitlab-update-for-release') } + {%- endif %} } + steps { build( + {%- if not gitlab_project %} job: '/debian/packages/{{name}}/update-for-release', + {%- else %} + job: '/debian/packages/{{name}}/gitlab-update-for-release', + {%- endif %} parameters: [ + {%- if not gitlab_project %} string(name: 'GIT_TAG', value: params.GIT_TAG), + {%- else %} + // remove refs/tags/ prefix from source branch + string(name: 'GIT_TAG', value: env.gitlabSourceBranch.substring(10)), + {%- endif %} ], - wait: false, + wait: true, ) } } + stage('Build Debian package') { when { + {%- if not gitlab_project %} expression { params.GIT_TAG ==~ /debian\/.*/ } expression { !(params.GIT_TAG ==~ /debian\/upstream\/.*/) } expression { jobExists('/debian/packages/{{name}}/gbp-buildpackage') } + {%- else %} + expression { "${env.gitlabSourceBranch}" ==~ /refs\/tags\/debian\/.*/ } + expression { !("${env.gitlabSourceBranch}" ==~ /refs\/tags\/debian\/upstream\/.*/) } + expression { jobExists('/debian/packages/{{name}}/gitlab-gbp-buildpackage') } + {%- endif %} } steps { build( + {%- if not gitlab_project %} job: '/debian/packages/{{name}}/gbp-buildpackage', parameters: [ string(name: 'GIT_REVISION', value: params.GIT_TAG), booleanParam(name: 'DO_UPLOAD', value: true), ], - wait: false, + {%- else %} + job: '/debian/packages/{{name}}/gitlab-gbp-buildpackage', + parameters: [ + // remove refs/tags/ prefix from source branch + string(name: 'GIT_REVISION', value: env.gitlabSourceBranch.substring(10)), + booleanParam(name: 'DO_UPLOAD', value: true), + ], + {%- endif %} + wait: true, ) } } - {%- endif %} } + {%- if gitlab_project %} + post { + failure { + updateGitlabCommitStatus name: 'jenkins', state: 'failed' + } + success { + updateGitlabCommitStatus name: 'jenkins', state: 'success' + } + aborted { + updateGitlabCommitStatus name: 'jenkins', state: 'canceled' + } + } + {%- endif %} } diff --git a/jobs/templates/incoming-tag.yaml b/jobs/templates/incoming-tag.yaml index ece06c3..2b1b89c 100644 --- a/jobs/templates/incoming-tag.yaml +++ b/jobs/templates/incoming-tag.yaml @@ -1,37 +1,40 @@ - job-template: &incoming_tag name: "{name}/incoming-tag" display-name: Incoming tag project-type: pipeline auth-token: "ph4br1cat0r" incoming_tag_auto_pypi_host: pypi.org sandbox: true properties: - build-discarder: num-to-keep: 20 parameters: - git-parameter: name: GIT_TAG description: git tag to process type: PT_TAG sortMode: DESCENDING_SMART selectedValue: TOP dsl: !include-jinja2: incoming-tag.groovy.j2 - job-template: name: "debian/deps/{name}/incoming-tag" dsl: !include-jinja2: dependency-incoming-tag.groovy.j2 <<: *incoming_tag - job-template: name: "{name}/gitlab-incoming-tag" display-name: Incoming tag (GitLab) gitlab_project: true parameters: + properties: + - gitlab: + connection: "{gitlab_connection_name}" triggers: - gitlab: trigger-push: true trigger-merge-request: false add-ci-message: true # secret jenkins token is generated when executing tox secret-token: !include-raw: jenkins-token <<: *incoming_tag diff --git a/jobs/templates/swh-pipeline.yaml b/jobs/templates/swh-pipeline.yaml index 0d8938e..3b43cda 100644 --- a/jobs/templates/swh-pipeline.yaml +++ b/jobs/templates/swh-pipeline.yaml @@ -1,62 +1,64 @@ - job-template: &master_tests name: "{name}/tests" display-name: "master branch" project-type: pipeline docker_image: tox cypress_docker_image: cypress sphinx_docker_image: sphinx default-tox-environment: py3 concurrent: true triggers: - timed: "@daily" sandbox: true auth-token: "ph4br1cat0r" properties: - build-discarder: days-to-keep: 90 artifact-num-to-keep: 20 phabricator_diff: false do_cypress: false timeout: 10 max_concurrent: 0 parameters: - git-parameter: name: REVISION type: PT_REVISION defaultValue: master description: Git revision to build. - string: name: PHID description: PHID of the Phabricator target object on which results will be reported. - string: name: TOX_ENVIRONMENT description: tox environment to use for the main tox run default: "{default-tox-environment}" dsl: !include-jinja2: swh-pipeline.groovy.j2 - job-template: name: "{name}/gitlab-tests" display-name: "GitLab builds" auth-token: properties: - gitlab: connection: "{gitlab_connection_name}" triggers: - gitlab: trigger-push: true trigger-merge-request: true add-ci-message: true cancel-pending-builds-on-update: true # secret jenkins token is generated when executing tox secret-token: !include-raw: jenkins-token + branch-filter-type: RegexBasedFilter + target-branch-regex: ^(?!(debian|pristine-tar)).* gitlab_project: true parameters: - string: name: REVISION description: Git revision to build. - string: name: TOX_ENVIRONMENT description: tox environment to use for the main tox run default: "{default-tox-environment}" <<: *master_tests diff --git a/jobs/tools/jenkins-jobs-builder.groovy.j2 b/jobs/tools/jenkins-jobs-builder.groovy.j2 index 60dbe74..8c815ed 100644 --- a/jobs/tools/jenkins-jobs-builder.groovy.j2 +++ b/jobs/tools/jenkins-jobs-builder.groovy.j2 @@ -1,59 +1,58 @@ pipeline { agent any stages { stage('Checkout Repository') { steps { updateGitlabCommitStatus name: 'jenkins', state: 'running' checkout([ $class: 'GitSCM', branches: [[name: env.gitlabSourceBranch ?: "master"]], userRemoteConfigs: [[ url: '{{gitlab_url}}/{{gitlab_project_name}}.git', refspec: '+refs/heads/*:refs/remotes/origin/* \ +refs/merge-requests/*/head:refs/remotes/origin/merge-requests/*' ]], ]) } } stage('Test Jenkins jobs can be generated') { steps { sh('tox') } } stage('Update Jenkins jobs and setup gitlab webhooks') { when { // really update jenkins jobs configuration when pushing commits to master // or when manually triggering the job from Jenkins UI - expression { !env.gitlabMergeRequestIid } anyOf { expression { !env.gitlabSourceBranch } expression { "${env.gitlabSourceBranch}" == "master" } } } steps { script { sh('tox -- update --delete-old') build( job: '/jenkins-tools/setup-gitlab-webhooks', ) } } } } post { failure { updateGitlabCommitStatus name: 'jenkins', state: 'failed' } success { updateGitlabCommitStatus name: 'jenkins', state: 'success' } aborted { updateGitlabCommitStatus name: 'jenkins', state: 'canceled' } } }