diff --git a/jobs/templates/debian/gbp-buildpackage.groovy b/jobs/templates/debian/gbp-buildpackage.groovy new file mode 100644 index 0000000..77ddbe8 --- /dev/null +++ b/jobs/templates/debian/gbp-buildpackage.groovy @@ -0,0 +1,272 @@ +def repo_name = '{display-name}' + +def changelog_distribution + +def repo_host = 'pergamon.internal.softwareheritage.org' +def repo_user = 'swhdebianrepo' +def repo_path = '/srv/softwareheritage/repository' + +def upload_target = "${{repo_path}}/incoming" +def repo_command = "umask 002; reprepro -vb ${{repo_path}} processincoming incoming" + +pipeline {{ + agent {{ label 'debian' }} + environment {{ + PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' + }} + stages {{ + stage('Notify Phabricator start') {{ + when {{ + beforeAgent true + expression {{ params.PHID }} + }} + agent {{ label 'swh-tox' }} + steps {{ + withCredentials([ + string(credentialsId: 'swh-public-ci', + variable: 'PHAB_CONDUIT_TOKEN')]) {{ + sh ''' + python3 -m pyarcanist send-message work $PHID + ''' + }} + }} + }} + stage('Checkout') {{ + steps {{ + cleanWs() + checkout([ + $class: 'GitSCM', + branches: [[name: params.GIT_REVISION]], + userRemoteConfigs: [[ + url: "https://forge.softwareheritage.org/source/${{repo_name}}.git", + ]], + extensions: [ + [$class: 'RelativeTargetDirectory', relativeTargetDir: repo_name], + ], + ]) + script {{ + dir(repo_name) {{ + if(!fileExists('debian/changelog')) {{ + error('Trying to build a debian package without a debian/changelog') + }} + + changelog_distribution = sh( + script: 'dpkg-parsechangelog -SDistribution', + returnStdout: true, + ).trim() + + def parsed_gbp_config = readProperties( + file: 'debian/gbp.conf', + defaults: ['debian-branch': 'master'], + ) + + def debian_branch = parsed_gbp_config['debian-branch'] + + sh """ + git checkout -b ${{debian_branch}} + git branch -f pristine-tar origin/pristine-tar + git branch -f debian/upstream origin/debian/upstream + """ + }} + }} + }} + }} + stage('Write extra keys') {{ + when {{ + beforeAgent true + expression {{ changelog_distribution != 'UNRELEASED' }} + }} + steps {{ + dir('keys') {{ + writeFile( + file: 'ceph.asc', + text: '''-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mQINBFX4hgkBEADLqn6O+UFp+ZuwccNldwvh5PzEwKUPlXKPLjQfXlQRig1flpCH +E0HJ5wgGlCtYd3Ol9f9+qU24kDNzfbs5bud58BeE7zFaZ4s0JMOMuVm7p8JhsvkU +C/Lo/7NFh25e4kgJpjvnwua7c2YrA44ggRb1QT19ueOZLK5wCQ1mR+0GdrcHRCLr +7Sdw1d7aLxMT+5nvqfzsmbDullsWOD6RnMdcqhOxZZvpay8OeuK+yb8FVQ4sOIzB +FiNi5cNOFFHg+8dZQoDrK3BpwNxYdGHsYIwU9u6DWWqXybBnB9jd2pve9PlzQUbO +eHEa4Z+jPqxY829f4ldaql7ig8e6BaInTfs2wPnHJ+606g2UH86QUmrVAjVzlLCm +nqoGymoAPGA4ObHu9X3kO8viMBId9FzooVqR8a9En7ZE0Dm9O7puzXR7A1f5sHoz +JdYHnr32I+B8iOixhDUtxIY4GA8biGATNaPd8XR2Ca1hPuZRVuIiGG9HDqUEtXhV +fY5qjTjaThIVKtYgEkWMT+Wet3DPPiWT3ftNOE907e6EWEBCHgsEuuZnAbku1GgD +LBH4/a/yo9bNvGZKRaTUM/1TXhM5XgVKjd07B4cChgKypAVHvef3HKfCG2U/DkyA +LjteHt/V807MtSlQyYaXUTGtDCrQPSlMK5TjmqUnDwy6Qdq8dtWN3DtBWQARAQAB +tCpDZXBoLmNvbSAocmVsZWFzZSBrZXkpIDxzZWN1cml0eUBjZXBoLmNvbT6JAjgE +EwECACIFAlX4hgkCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEOhKwsBG +DzmUXdIQAI8YPcZMBWdv489q8CzxlfRIRZ3Gv/G/8CH+EOExcmkVZ89mVHngCdAP +DOYCl8twWXC1lwJuLDBtkUOHXNuR5+Jcl5zFOUyldq1Hv8u03vjnGT7lLJkJoqpG +l9QD8nBqRvBU7EM+CU7kP8+09b+088pULil+8x46PwgXkvOQwfVKSOr740Q4J4nm +/nUOyTNtToYntmt2fAVWDTIuyPpAqA6jcqSOC7Xoz9cYxkVWnYMLBUySXmSS0uxl +3p+wK0lMG0my/gb+alke5PAQjcE5dtXYzCn+8Lj0uSfCk8Gy0ZOK2oiUjaCGYN6D +u72qDRFBnR3jaoFqi03bGBIMnglGuAPyBZiI7LJgzuT9xumjKTJW3kN4YJxMNYu1 +FzmIyFZpyvZ7930vB2UpCOiIaRdZiX4Z6ZN2frD3a/vBxBNqiNh/BO+Dex+PDfI4 +TqwF8zlcjt4XZ2teQ8nNMR/D8oiYTUW8hwR4laEmDy7ASxe0p5aijmUApWq5UTsF ++s/QbwugccU0iR5orksM5u9MZH4J/mFGKzOltfGXNLYI6D5Mtwrnyi0BsF5eY0u6 +vkdivtdqrq2DXY+ftuqLOQ7b+t1RctbcMHGPptlxFuN9ufP5TiTWSpfqDwmHCLsT +k2vFiMwcHdLpQ1IH8ORVRgPPsiBnBOJ/kIiXG2SxPUTjjEGOVgeA +=/Tod +-----END PGP PUBLIC KEY BLOCK----- +''' + ) + }} + }} + }} + stage('Build package') {{ + when {{ + beforeAgent true + expression {{ changelog_distribution != 'UNRELEASED' }} + }} + steps {{ + script {{ + dir (repo_name) {{ + def wanted_extra_repositories = params.EXTRA_REPOSITORIES.split(',') + + def build_dep_resolver + + def extra_repositories = [] + def extra_repository_keys = [] + + def base_distribution = changelog_distribution.split('-')[0] + def backports = changelog_distribution.endsWith('-backports') + def swh = changelog_distribution.endsWith('-swh') + + if (base_distribution in ['unstable', 'experimental'] || 'incoming' in wanted_extra_repositories) {{ + def suites = [] + if (base_distribution == 'unstable') {{ + suites = ['buildd-unstable'] + }} else if (base_distribution == 'experimental') {{ + suites = ['buildd-unstable', 'buildd-experimental'] + }} else {{ + suites = ["buildd-${{base_distribution}}-proposed-updates"] + if (backports || swh) {{ + suites.add("buildd-${{base_distribution}}-backports") + }} + }} + suites.each {{suite -> + extra_repositories.add("deb http://incoming.debian.org/debian-buildd/ ${{suite}} main") + }} + }} + + if (swh || 'swh' in wanted_extra_repositories) {{ + def swh_distribution = "${{base_distribution}}-swh" + if (base_distribution in ['unstable', 'experimental']) {{ + swh_distribution = 'unstable' + }} + extra_repositories.add("deb [trusted=yes] https://debian.softwareheritage.org/ ${{swh_distribution}} main") + }} + + if ((backports || swh || 'backports' in wanted_extra_repositories) && !(base_distribution in ['unstable', 'experimental'])) {{ + extra_repositories.add("deb http://deb.debian.org/debian/ ${{base_distribution}}-backports main") + build_dep_resolver = 'aptitude' + }} + + if ('ceph' in wanted_extra_repositories) {{ + extra_repositories.add("deb https://download.ceph.com/debian-luminous/ ${{base_distribution}} main") + extra_repository_keys.add('../keys/ceph.asc') + }} + + if (params.BUILD_DEP_RESOLVER) {{ + build_dep_resolver = params.BUILD_DEP_RESOLVER + }} + + def gbp_buildpackage = [ + 'gbp buildpackage', + '--git-builder=sbuild', + '--nolog', + '--batch', + '--no-clean-source', + '--no-run-lintian', + '--arch-all', + '--source', + '--force-orig-source', + ] + + if (build_dep_resolver != null) {{ + gbp_buildpackage.add("--build-dep-resolver=${{build_dep_resolver}}") + }} + + extra_repositories.each {{ repo -> + gbp_buildpackage.add("--extra-repository='${{repo}}'") + }} + + extra_repository_keys.each {{ key -> + gbp_buildpackage.add("--extra-repository-key='${{key}}'") + }} + + def gbp_buildpackage_cmd = gbp_buildpackage.join(' ') + + sh(script: gbp_buildpackage_cmd) + }} + + if (params.DO_UPLOAD) {{ + sh(script: 'debsign *.changes') + }} + + archiveArtifacts( + artifacts: sh( + script: 'dcmd echo *.changes', + returnStdout: true + ).split().join(','), + fingerprint: true, + ) + }} + }} + }} + stage('Upload package') {{ + when {{ + beforeAgent true + expression {{ changelog_distribution != 'UNRELEASED' }} + expression {{ params.DO_UPLOAD }} + }} + steps {{ + sshagent (credentials: ['jenkins-debian-repo-ssh']) {{ + sh """ + dcmd rsync -v *.changes ${{repo_user}}@${{repo_host}}:${{upload_target}} + ssh ${{repo_user}}@${{repo_host}} '${{repo_command}}' + """ + }} + }} + }} + stage('Tag package') {{ + when {{ + beforeAgent true + expression {{ changelog_distribution != 'UNRELEASED' }} + expression {{ params.DO_TAG }} + }} + steps {{ + dir (repo_name) {{ + sh 'gbp buildpackage --git-tag-only' + sshagent (credentials: ['jenkins-public-ci-ssh']) {{ + sh 'git push --tags' + }} + }} + }} + }} + }} + post {{ + always {{ + node('swh-tox') {{ + withCredentials([ + string(credentialsId: 'swh-public-ci', + variable: 'PHAB_CONDUIT_TOKEN')]) {{ + withEnv(["JOBSTATUS=${{currentBuild.currentResult}}"]) {{ + sh ''' + if [ "$JOBSTATUS" = "SUCCESS" ]; then + MSGTYPE=pass + else + MSGTYPE=fail + fi + echo "Current job status is $JOBSTATUS -> $MGSTYPE" + if [ -n "$PHID" ]; then + python3 -m pyarcanist send-message $MSGTYPE $PHID + fi + ''' + }} + }} + }} + }} + }} +}} diff --git a/jobs/templates/debian/gbp-buildpackage.yaml b/jobs/templates/debian/gbp-buildpackage.yaml new file mode 100644 index 0000000..ecc89f2 --- /dev/null +++ b/jobs/templates/debian/gbp-buildpackage.yaml @@ -0,0 +1,53 @@ +- job-template: + name: 'debian/packages/{name}/gbp-buildpackage' + description: | + Build Debian package + auth-token: 'ph4br1cat0r' + deb-extra-repositories: + project-type: pipeline + parameters: + - string: + name: GIT_REVISION + default: debian/unstable-swh + description: Git revision to build + - string: + name: PHID + description: PHID of the phabricator target object on which results will be reported + - extended-choice: + name: EXTRA_REPOSITORIES + description: extra repositories needed to build this package + type: checkbox + value: !join: + - ',' + - + - incoming + - backports + - swh + - ceph + value-description: !join: + - ',' + - + - The buildd incoming repository (default for builds from unstable) + - The debian backports repository (default for builds for stable-swh) + - The Software Heritage debian repository (default for builds for swh branches) + - The ceph repository + default-value: '{deb-extra-repositories}' + - choice: + name: BUILD_DEP_RESOLVER + description: | + the sbuild build dependency resolver (overrides the automatic detection) + choices: + - '' + - apt + - aptitude + - bool: + name: DO_UPLOAD + description: upload the built package + default: false + - bool: + name: DO_TAG + description: tag the repository once the package has been built + default: false + dsl: + !include-raw: gbp-buildpackage.groovy + diff --git a/jobs/templates/debian/update-for-release.groovy b/jobs/templates/debian/update-for-release.groovy index 5f21839..9c3bb01 100644 --- a/jobs/templates/debian/update-for-release.groovy +++ b/jobs/templates/debian/update-for-release.groovy @@ -1,156 +1,156 @@ def upstream_tag = params.GIT_TAG def version = upstream_tag.substring(1) def repo_name = '{display-name}' def python_module = repo_name.replace('-', '.') def full_environ def debian_upstream_tag_exists = false pipeline {{ agent {{ label 'debian' }} stages {{ stage('Checkout') {{ steps {{ cleanWs() checkout([ $class: 'GitSCM', branches: [[name: 'debian/unstable-swh'], [name: 'debian/*'], [name: 'pristine-tar']], userRemoteConfigs: [[ url: "https://forge.softwareheritage.org/source/${{repo_name}}.git", ]], extensions: [ [$class: 'RelativeTargetDirectory', relativeTargetDir: repo_name], ], poll: false, ]) dir (repo_name) {{ sh ''' git checkout -B pristine-tar origin/pristine-tar git checkout -B debian/upstream origin/debian/upstream git checkout -B debian/unstable-swh origin/debian/unstable-swh ''' script {{ def tag_exists_retval = sh( script: "git rev-parse --verify debian/upstream/${{version}}", returnStatus: true ) debian_upstream_tag_exists = (tag_exists_retval == 0) }} }} }} }} stage('Get author information from tag') {{ when {{ beforeAgent true expression {{ !debian_upstream_tag_exists }} }} steps {{ dir (repo_name) {{ // This script retrieves the author data for the tag that we're currently processing script {{ // Check if the tag is annotated or not def objecttype = sh( script: "git tag -l --format='%(*objecttype)' ${{upstream_tag}}", returnStdout: true, ).trim(); def tagger = !objecttype.isEmpty() ? 'tagger' : 'author'; def tagger_name = sh( script: "git tag -l --format='%(${{tagger}}name)' ${{upstream_tag}}", returnStdout: true, ).trim(); def tagger_email = sh( script: "git tag -l --format='%(${{tagger}}email)' ${{upstream_tag}}", returnStdout: true, ).trim(); tagger_email = tagger_email.substring(1, tagger_email.length() - 1); def tagger_date = sh( - script: "git tag -l --format='%(${{tagger}}date)' ${{upstream_tag}}", + script: "git tag -l --format='%(${{tagger}}date:iso)' ${{upstream_tag}}", returnStdout: true, ).trim(); def hostname = sh( script: "hostname --fqdn", returnStdout: true, ).trim(); def short_hostname = hostname - '.internal.softwareheritage.org'; full_environ = [ "DEBEMAIL=jenkins@${{hostname}}", "DEBFULLNAME=Software Heritage autobuilder (on ${{short_hostname}})", "UPSTREAM_TAG=${{upstream_tag}}", "UPSTREAM_TAGGER_NAME=${{tagger_name}}", "UPSTREAM_TAGGER_EMAIL=${{tagger_email}}", "UPSTREAM_TAGGER_DATE=${{tagger_date}}", "UPSTREAM_VERSION=${{version}}", "PYTHON_MODULE=${{python_module}}", ]; }} }} }} }} stage('gbp import-orig') {{ when {{ beforeAgent true expression {{ !debian_upstream_tag_exists }} }} steps {{ copyArtifacts( projectName: '/{name}/pypi-upload', parameters: 'GIT_TAG=' + params.GIT_TAG, ) withEnv(full_environ) {{ dir (repo_name) {{ sh ''' gbp import-orig --merge-mode=replace --no-symlink-orig -u ${{UPSTREAM_VERSION}} ../dist/${{PYTHON_MODULE}}-${{UPSTREAM_VERSION}}.tar.gz ''' }} }} }} }} stage('unstable-swh changelog') {{ when {{ beforeAgent true expression {{ !debian_upstream_tag_exists }} }} steps {{ withEnv(full_environ) {{ dir(repo_name) {{ sh ''' dch -v ${{UPSTREAM_VERSION}}-1~swh1 "New upstream release ${{UPSTREAM_VERSION}}\n - (tagged by ${{UPSTREAM_TAGGER_NAME}} <${{UPSTREAM_TAGGER_EMAIL}}> on ${{UPSTREAM_TAGGER_DATE}})" dch "Upstream changes:\n$(git tag -l --format='%(contents:subject)%0a%(contents:body)' ${{UPSTREAM_TAG}} | sed -E -e '/^$/d' -e 's/^ *(- *)?//' -e 's/^/ - /')" dch -D unstable-swh --force-distribution '' git add debian/changelog git commit --no-verify -m "Updated debian changelog for version ${{UPSTREAM_VERSION}}" git show ''' }} }} }} }} stage('Upload changes') {{ when {{ beforeAgent true expression {{ !debian_upstream_tag_exists }} }} steps {{ dir (repo_name) {{ sshagent (credentials: ['jenkins-public-ci-ssh']) {{ script {{ def git_push = 'git push --follow-tags --all' if (params.DRY_RUN) {{ git_push += ' -n' }} sh(script: git_push) }} }} }} }} }} }} }}