diff --git a/jobs/debian/folders.yaml b/jobs/debian/folders.yaml
index 1f0d538..33883ba 100644
--- a/jobs/debian/folders.yaml
+++ b/jobs/debian/folders.yaml
@@ -1,37 +1,49 @@
- job:
name: 'debian'
display-name: 'Debian'
project-type: folder
raw:
xml: |
false
- job:
name: 'debian/maintenance'
display-name: 'Debian Maintenance'
description: |
Debian package building-related maintenance jobs.
project-type: folder
raw:
xml: |
false
- job:
name: 'debian/packages'
display-name: 'Debian Packages'
project-type: folder
raw:
xml: |
false
+
+- job:
+ name: 'debian/deps'
+ display-name: 'Debian Dependency Packages'
+ project-type: folder
+ raw:
+ xml: |
+
+
+ false
+
+
diff --git a/jobs/dependency-packages.yaml b/jobs/dependency-packages.yaml
new file mode 100644
index 0000000..1276c9b
--- /dev/null
+++ b/jobs/dependency-packages.yaml
@@ -0,0 +1,14 @@
+- job-group:
+ name: 'dependency-jobs-{name}'
+ jobs:
+ - "debian/deps/{name}"
+ - "debian/deps/{name}/incoming-tag"
+ - "debian/deps/{name}/gbp-buildpackage"
+ - "debian/deps/{name}/automatic-backport"
+
+- project:
+ name: XPMK
+ display-name: python3-mirakuru
+ pkg: mirakuru
+ jobs:
+ - 'dependency-jobs-{name}'
diff --git a/jobs/templates/debian/automatic-backport.yaml b/jobs/templates/debian/automatic-backport.yaml
index a4c49c4..621100b 100644
--- a/jobs/templates/debian/automatic-backport.yaml
+++ b/jobs/templates/debian/automatic-backport.yaml
@@ -1,39 +1,43 @@
-- job-template:
+- job-template: &automatic_backport
name: 'debian/packages/{name}/automatic-backport'
description: |
Backport Debian package to the given release
project-type: pipeline
parameters:
- git-parameter:
name: GIT_TAG
description: git tag to backport
type: PT_TAG
sortMode: DESCENDING_SMART
selectedValue: TOP
tagFilter: debian/.*
- choice:
name: SOURCE
description: check that the source branch corresponds to this before issuing the backport
choices:
- unstable-swh
- unstable
- bookworm-swh
- bullseye-swh
- buster-swh
- stretch-swh
default: unstable-swh
- choice:
name: DESTINATION
description: the destination branch the backport targets
choices:
- stretch-swh
- buster-swh
- bullseye-swh
- bookworm-swh
default: stretch-swh
- bool:
name: DRY_RUN
description: Don't push the resulting changes to a Debian branch
default: false
dsl:
!include-raw: automatic-backport.groovy
+
+- job-template:
+ name: 'debian/deps/{name}/automatic-backport'
+ <<: *automatic_backport
diff --git a/jobs/templates/debian/folder.yaml b/jobs/templates/debian/folder.yaml
index 7921791..2ada3df 100644
--- a/jobs/templates/debian/folder.yaml
+++ b/jobs/templates/debian/folder.yaml
@@ -1,5 +1,9 @@
- job-template:
name: 'debian/packages/{name}'
project-type: folder
display-name: '{display-name}'
+- job-template:
+ name: 'debian/deps/{name}'
+ project-type: folder
+ display-name: '{display-name}'
diff --git a/jobs/templates/debian/gbp-buildpackage.groovy b/jobs/templates/debian/gbp-buildpackage.groovy
index 1cfb3b1..eb4e9a6 100644
--- a/jobs/templates/debian/gbp-buildpackage.groovy
+++ b/jobs/templates/debian/gbp-buildpackage.groovy
@@ -1,294 +1,296 @@
def repo_name = '{display-name}'
def changelog_distribution
def repo_host = 'pergamon.internal.softwareheritage.org'
def repo_user = 'swhdebianrepo'
def repo_path = '/srv/softwareheritage/repository'
def upload_target = "${{repo_path}}/incoming"
def repo_command = "umask 002; reprepro -vb ${{repo_path}} processincoming incoming"
+def backport_job = '/' + (currentBuild.fullProjectName.split('/')[0..-2] + ['automatic-backport']).join('/')
+
pipeline {{
agent {{ label 'debian' }}
environment {{
PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/'
}}
stages {{
stage('Notify Phabricator start') {{
when {{
beforeAgent true
expression {{ params.PHID }}
}}
agent {{ label 'swh-tox' }}
steps {{
withCredentials([
string(credentialsId: 'swh-public-ci',
variable: 'PHAB_CONDUIT_TOKEN')]) {{
sh '''
python3 -m pyarcanist send-message work $PHID
'''
}}
}}
}}
stage('Checkout') {{
steps {{
cleanWs()
checkout([
$class: 'GitSCM',
branches: [[name: params.GIT_REVISION]],
userRemoteConfigs: [[
url: "https://forge.softwareheritage.org/source/${{repo_name}}.git",
]],
extensions: [
[$class: 'RelativeTargetDirectory', relativeTargetDir: repo_name],
],
])
script {{
dir(repo_name) {{
if(!fileExists('debian/changelog')) {{
error('Trying to build a debian package without a debian/changelog')
}}
changelog_distribution = sh(
script: 'dpkg-parsechangelog -SDistribution',
returnStdout: true,
).trim()
def parsed_gbp_config = readProperties(
file: 'debian/gbp.conf',
defaults: ['debian-branch': 'master'],
)
def debian_branch = parsed_gbp_config['debian-branch']
sh """
git checkout -b ${{debian_branch}}
git branch -f pristine-tar origin/pristine-tar
git branch -f debian/upstream origin/debian/upstream
"""
}}
}}
}}
}}
stage('Write extra keys') {{
when {{
beforeAgent true
expression {{ changelog_distribution != 'UNRELEASED' }}
}}
steps {{
dir('keys') {{
writeFile(
file: 'ceph.asc',
text: '''-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1
mQINBFX4hgkBEADLqn6O+UFp+ZuwccNldwvh5PzEwKUPlXKPLjQfXlQRig1flpCH
E0HJ5wgGlCtYd3Ol9f9+qU24kDNzfbs5bud58BeE7zFaZ4s0JMOMuVm7p8JhsvkU
C/Lo/7NFh25e4kgJpjvnwua7c2YrA44ggRb1QT19ueOZLK5wCQ1mR+0GdrcHRCLr
7Sdw1d7aLxMT+5nvqfzsmbDullsWOD6RnMdcqhOxZZvpay8OeuK+yb8FVQ4sOIzB
FiNi5cNOFFHg+8dZQoDrK3BpwNxYdGHsYIwU9u6DWWqXybBnB9jd2pve9PlzQUbO
eHEa4Z+jPqxY829f4ldaql7ig8e6BaInTfs2wPnHJ+606g2UH86QUmrVAjVzlLCm
nqoGymoAPGA4ObHu9X3kO8viMBId9FzooVqR8a9En7ZE0Dm9O7puzXR7A1f5sHoz
JdYHnr32I+B8iOixhDUtxIY4GA8biGATNaPd8XR2Ca1hPuZRVuIiGG9HDqUEtXhV
fY5qjTjaThIVKtYgEkWMT+Wet3DPPiWT3ftNOE907e6EWEBCHgsEuuZnAbku1GgD
LBH4/a/yo9bNvGZKRaTUM/1TXhM5XgVKjd07B4cChgKypAVHvef3HKfCG2U/DkyA
LjteHt/V807MtSlQyYaXUTGtDCrQPSlMK5TjmqUnDwy6Qdq8dtWN3DtBWQARAQAB
tCpDZXBoLmNvbSAocmVsZWFzZSBrZXkpIDxzZWN1cml0eUBjZXBoLmNvbT6JAjgE
EwECACIFAlX4hgkCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEOhKwsBG
DzmUXdIQAI8YPcZMBWdv489q8CzxlfRIRZ3Gv/G/8CH+EOExcmkVZ89mVHngCdAP
DOYCl8twWXC1lwJuLDBtkUOHXNuR5+Jcl5zFOUyldq1Hv8u03vjnGT7lLJkJoqpG
l9QD8nBqRvBU7EM+CU7kP8+09b+088pULil+8x46PwgXkvOQwfVKSOr740Q4J4nm
/nUOyTNtToYntmt2fAVWDTIuyPpAqA6jcqSOC7Xoz9cYxkVWnYMLBUySXmSS0uxl
3p+wK0lMG0my/gb+alke5PAQjcE5dtXYzCn+8Lj0uSfCk8Gy0ZOK2oiUjaCGYN6D
u72qDRFBnR3jaoFqi03bGBIMnglGuAPyBZiI7LJgzuT9xumjKTJW3kN4YJxMNYu1
FzmIyFZpyvZ7930vB2UpCOiIaRdZiX4Z6ZN2frD3a/vBxBNqiNh/BO+Dex+PDfI4
TqwF8zlcjt4XZ2teQ8nNMR/D8oiYTUW8hwR4laEmDy7ASxe0p5aijmUApWq5UTsF
+s/QbwugccU0iR5orksM5u9MZH4J/mFGKzOltfGXNLYI6D5Mtwrnyi0BsF5eY0u6
vkdivtdqrq2DXY+ftuqLOQ7b+t1RctbcMHGPptlxFuN9ufP5TiTWSpfqDwmHCLsT
k2vFiMwcHdLpQ1IH8ORVRgPPsiBnBOJ/kIiXG2SxPUTjjEGOVgeA
=/Tod
-----END PGP PUBLIC KEY BLOCK-----
'''
)
}}
}}
}}
stage('Build package') {{
when {{
beforeAgent true
expression {{ changelog_distribution != 'UNRELEASED' }}
}}
steps {{
script {{
dir (repo_name) {{
def wanted_extra_repositories = params.EXTRA_REPOSITORIES.split(',')
def build_dep_resolver
def extra_repositories = []
def extra_repository_keys = []
def base_distribution = changelog_distribution.split('-')[0]
def backports = changelog_distribution.endsWith('-backports')
def swh = changelog_distribution.endsWith('-swh')
if (base_distribution in ['unstable', 'experimental'] || 'incoming' in wanted_extra_repositories) {{
def suites = []
if (base_distribution == 'unstable') {{
suites = ['buildd-unstable']
}} else if (base_distribution == 'experimental') {{
suites = ['buildd-unstable', 'buildd-experimental']
}} else {{
suites = ["buildd-${{base_distribution}}-proposed-updates"]
if (backports || swh) {{
suites.add("buildd-${{base_distribution}}-backports")
}}
}}
suites.each {{suite ->
extra_repositories.add("deb http://incoming.debian.org/debian-buildd/ ${{suite}} main")
}}
}}
if (swh || 'swh' in wanted_extra_repositories) {{
def swh_distribution = "${{base_distribution}}-swh"
if (base_distribution in ['unstable', 'experimental']) {{
swh_distribution = 'unstable'
}}
extra_repositories.add("deb [trusted=yes] https://debian.softwareheritage.org/ ${{swh_distribution}} main")
}}
if ((backports || swh || 'backports' in wanted_extra_repositories) && !(base_distribution in ['unstable', 'experimental'])) {{
extra_repositories.add("deb http://deb.debian.org/debian/ ${{base_distribution}}-backports main")
build_dep_resolver = 'aptitude'
}}
if ('ceph' in wanted_extra_repositories && !(base_distribution in ['unstable', 'experimental'])) {{
extra_repositories.add("deb https://download.ceph.com/debian-luminous/ ${{base_distribution}} main")
extra_repository_keys.add('../keys/ceph.asc')
}}
if (params.BUILD_DEP_RESOLVER) {{
build_dep_resolver = params.BUILD_DEP_RESOLVER
}}
def hostname = sh(
script: "hostname --fqdn",
returnStdout: true,
).trim();
def short_hostname = hostname - '.internal.softwareheritage.org';
def uploader = "Software Heritage autobuilder (on ${{short_hostname}}) "
def gbp_buildpackage = [
'gbp buildpackage',
'--git-builder=sbuild',
'--nolog',
'--batch',
'--no-clean-source',
'--no-run-lintian',
'--arch-all',
'--source',
'--force-orig-source',
"--uploader='${{uploader}}'",
]
if (build_dep_resolver != null) {{
gbp_buildpackage.add("--build-dep-resolver=${{build_dep_resolver}}")
}}
extra_repositories.each {{ repo ->
gbp_buildpackage.add("--extra-repository='${{repo}}'")
}}
extra_repository_keys.each {{ key ->
gbp_buildpackage.add("--extra-repository-key='${{key}}'")
}}
def gbp_buildpackage_cmd = gbp_buildpackage.join(' ')
sh(script: gbp_buildpackage_cmd)
}}
if (params.DO_UPLOAD) {{
sh(script: 'debsign *.changes')
}}
archiveArtifacts(
artifacts: sh(
script: 'dcmd echo *.changes',
returnStdout: true
).split().join(','),
fingerprint: true,
)
}}
}}
}}
stage('Upload package') {{
when {{
beforeAgent true
expression {{ changelog_distribution != 'UNRELEASED' }}
expression {{ params.DO_UPLOAD }}
}}
steps {{
sshagent (credentials: ['jenkins-debian-repo-ssh']) {{
sh """
dcmd rsync -v *.changes ${{repo_user}}@${{repo_host}}:${{upload_target}}
ssh ${{repo_user}}@${{repo_host}} '${{repo_command}}'
"""
}}
}}
}}
stage('Prepare backport') {{
when {{
beforeAgent true
expression {{ changelog_distribution != 'UNRELEASED' }}
expression {{ params.BACKPORT_ON_SUCCESS }}
- expression {{ jobExists('/debian/packages/{name}/automatic-backport') }}
+ expression {{ jobExists(backport_job) }}
}}
steps {{
script {{
params.BACKPORT_ON_SUCCESS.split(',').each {{ bpo_pair ->
def (src_suite, dst_suite) = bpo_pair.split('>')
if (src_suite == changelog_distribution) {{
build(
- job: '/debian/packages/{name}/automatic-backport',
+ job: backport_job,
parameters: [
string(name: 'GIT_TAG', value: params.GIT_REVISION),
string(name: 'SOURCE', value: src_suite),
string(name: 'DESTINATION', value: dst_suite),
],
wait: false,
)
}}
}}
}}
}}
}}
}}
post {{
always {{
node('swh-tox') {{
withCredentials([
string(credentialsId: 'swh-public-ci',
variable: 'PHAB_CONDUIT_TOKEN')]) {{
withEnv(["JOBSTATUS=${{currentBuild.currentResult}}"]) {{
sh '''
if [ "$JOBSTATUS" = "SUCCESS" ]; then
MSGTYPE=pass
else
MSGTYPE=fail
fi
echo "Current job status is $JOBSTATUS -> $MGSTYPE"
if [ -n "$PHID" ]; then
python3 -m pyarcanist send-message $MSGTYPE $PHID
fi
'''
}}
}}
}}
}}
}}
}}
diff --git a/jobs/templates/debian/gbp-buildpackage.yaml b/jobs/templates/debian/gbp-buildpackage.yaml
index bbcfa3e..8c22edc 100644
--- a/jobs/templates/debian/gbp-buildpackage.yaml
+++ b/jobs/templates/debian/gbp-buildpackage.yaml
@@ -1,61 +1,64 @@
-- job-template:
+- job-template: &gbp_buildpackage
name: 'debian/packages/{name}/gbp-buildpackage'
description: |
Build Debian package
auth-token: 'ph4br1cat0r'
deb-extra-repositories:
deb-backport-on-success: unstable-swh>stretch-swh
project-type: pipeline
parameters:
- string:
name: GIT_REVISION
default: debian/unstable-swh
description: Git revision to build
- string:
name: PHID
description: PHID of the phabricator target object on which results will be reported
- extended-choice:
name: EXTRA_REPOSITORIES
description: extra repositories needed to build this package
type: checkbox
value: !join:
- ','
-
- incoming
- backports
- swh
- ceph
value-description: !join:
- ','
-
- The buildd incoming repository (default for builds from unstable)
- The debian backports repository (default for builds for stable-swh)
- The Software Heritage debian repository (default for builds for swh branches)
- The ceph repository
default-value: '{deb-extra-repositories}'
- choice:
name: BUILD_DEP_RESOLVER
description: |
the sbuild build dependency resolver (overrides the automatic detection)
choices:
- ''
- apt
- aptitude
- bool:
name: DO_UPLOAD
description: upload the built package
default: false
- extended-choice:
name: BACKPORT_ON_SUCCESS
description: If the build is successful, which backports should we perform?
type: checkbox
value: !join:
- ','
-
- unstable-swh>stretch-swh
- unstable-swh>buster-swh
- buster-swh>stretch-swh
default-value: '{deb-backport-on-success}'
dsl:
!include-raw: gbp-buildpackage.groovy
+- job-template:
+ name: 'debian/deps/{name}/gbp-buildpackage'
+ <<: *gbp_buildpackage
diff --git a/jobs/templates/dependency-incoming-tag.groovy b/jobs/templates/dependency-incoming-tag.groovy
new file mode 100644
index 0000000..1be5b67
--- /dev/null
+++ b/jobs/templates/dependency-incoming-tag.groovy
@@ -0,0 +1,41 @@
+pipeline {{
+ agent none
+ stages {{
+ stage('Refresh tag list') {{
+ agent any
+ steps {{
+ checkout([
+ $class: 'GitSCM',
+ userRemoteConfigs: [[
+ url: 'https://forge.softwareheritage.org/source/{display-name}.git',
+ ]],
+ branches: [[
+ name: params.GIT_TAG,
+ ]],
+ browser: [
+ $class: 'Phabricator',
+ repo: '{display-name}',
+ repoUrl: 'https://forge.softwareheritage.org/',
+ ],
+ ])
+ }}
+ }}
+ stage('Build Debian package') {{
+ when {{
+ expression {{ params.GIT_TAG ==~ /debian\/.*/ }}
+ expression {{ !(params.GIT_TAG ==~ /debian\/upstream\/.*/) }}
+ expression {{ jobExists('/debian/deps/{name}/gbp-buildpackage') }}
+ }}
+ steps {{
+ build(
+ job: '/debian/deps/{name}/gbp-buildpackage',
+ parameters: [
+ string(name: 'GIT_REVISION', value: params.GIT_TAG),
+ booleanParam(name: 'DO_UPLOAD', value: true),
+ ],
+ wait: false,
+ )
+ }}
+ }}
+ }}
+}}
diff --git a/jobs/templates/incoming-tag.yaml b/jobs/templates/incoming-tag.yaml
index 45a76ef..27764ed 100644
--- a/jobs/templates/incoming-tag.yaml
+++ b/jobs/templates/incoming-tag.yaml
@@ -1,16 +1,22 @@
-- job-template:
+- job-template: &incoming_tag
name: "{name}/incoming-tag"
display-name: Incoming tag
project-type: pipeline
auth-token: 'ph4br1cat0r'
incoming-tag-auto-pypi-host: pypi.org
sandbox: true
parameters:
- git-parameter:
name: GIT_TAG
description: git tag to process
type: PT_TAG
sortMode: DESCENDING_SMART
selectedValue: TOP
dsl:
!include-raw: incoming-tag.groovy
+
+- job-template:
+ name: "debian/deps/{name}/incoming-tag"
+ dsl:
+ !include-raw: dependency-incoming-tag.groovy
+ <<: *incoming_tag