diff --git a/jobs/swh-packages.yaml b/jobs/swh-packages.yaml index 0bf8098..1cac17c 100644 --- a/jobs/swh-packages.yaml +++ b/jobs/swh-packages.yaml @@ -1,211 +1,234 @@ - job-group: name: 'swh-jobs-{name}' jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - '{name}/tests-on-diff' - "{name}/tox" - "debian/packages/{name}" - "debian/packages/{name}/update-for-release" - "debian/packages/{name}/automatic-backport" - "debian/packages/{name}/gbp-buildpackage" - project: name: DCORE display-name: swh-core repo_name: swh-core pkg: core + python_module: swh.core jobs: - 'swh-jobs-{name}' - project: name: DDEP display-name: swh-deposit repo_name: swh-deposit pkg: deposit + python_module: swh.deposit jobs: - 'swh-jobs-{name}' - project: name: DGRPH display-name: swh-graph repo_name: swh-graph pkg: graph + python_module: swh.graph jobs: - 'swh-jobs-{name}' - project: name: DCIDX display-name: swh-indexer repo_name: swh-indexer pkg: indexer + python_module: swh.indexer jobs: - 'swh-jobs-{name}' - project: name: DICP display-name: swh-icinga-plugins repo_name: swh-icinga-plugins pkg: icinga-plugins + python_module: swh.icinga_plugins jobs: - 'swh-jobs-{name}' - project: name: DJNL display-name: swh-journal repo_name: swh-journal docker_image: kafka pkg: journal + python_module: swh.journal jobs: - 'swh-jobs-{name}' - project: name: DLS display-name: swh-lister repo_name: swh-lister pkg: lister + python_module: swh.lister jobs: - 'swh-jobs-{name}' - project: name: DLDBASE display-name: swh-loader-core repo_name: swh-loader-core pkg: loader.core + python_module: swh.loader.core jobs: - 'swh-jobs-{name}' - project: name: DLDG display-name: swh-loader-git repo_name: swh-loader-git pkg: loader.git + python_module: swh.loader.git jobs: - 'swh-jobs-{name}' - project: name: DLDHG display-name: swh-loader-mercurial repo_name: swh-loader-mercurial pkg: loader.mercurial + python_module: swh.loader.mercurial jobs: - 'swh-jobs-{name}' - project: name: DLDSVN display-name: swh-loader-svn repo_name: swh-loader-svn pkg: loader.svn + python_module: swh.loader.svn jobs: - 'swh-jobs-{name}' - project: name: DMOD display-name: swh-model repo_name: swh-model pkg: model + python_module: swh.model jobs: - 'swh-jobs-{name}' - project: name: DOBJS display-name: swh-objstorage repo_name: swh-objstorage pkg: objstorage deb-extra-repositories: ceph + python_module: swh.objstorage jobs: - 'swh-jobs-{name}' - project: name: DOBJSRPL display-name: swh-objstorage-replayer repo_name: swh-objstorage-replayer pkg: objstorage.replayer + python_module: swh.objstorage.replayer jobs: - 'swh-jobs-{name}' - project: name: DSEA display-name: swh-search repo_name: swh-search pkg: search deb-extra-repositories: elasticsearch + python_module: swh.search jobs: - 'swh-jobs-{name}' - project: name: TSBX display-name: swh-sandbox repo_name: swh-sandbox pkg: sandbox incoming_tag_auto_pypi_host: test.pypi.org + python_module: swh.sandbox jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - '{name}/tests-on-diff' - "{name}/tox" - project: name: DTSCN display-name: swh-scanner repo_name: swh-scanner pkg: scanner + python_module: swh.scanner jobs: - 'swh-jobs-{name}' - project: name: DSCH display-name: swh-scheduler repo_name: swh-scheduler pkg: scheduler + python_module: swh.scheduler jobs: - 'swh-jobs-{name}' - project: name: DSTO display-name: swh-storage repo_name: swh-storage pkg: storage deb-extra-repositories: cassandra + python_module: swh.storage jobs: - 'swh-jobs-{name}' - project: name: DVAU display-name: swh-vault repo_name: swh-vault pkg: vault + python_module: swh.vault jobs: - 'swh-jobs-{name}' - project: name: DWAPPS display-name: swh-web repo_name: swh-web pkg: web + python_module: swh.web do_cypress: true jobs: - 'swh-jobs-{name}' - '{name}/cypress' - '{name}/cypress-diff' - project: name: DWCLI display-name: swh-web-client repo_name: swh-web-client pkg: web-client + python_module: swh.web.client jobs: - 'swh-jobs-{name}' - project: name: DFUSE display-name: swh-fuse repo_name: swh-fuse pkg: fuse + python_module: swh.fuse docker_options: --privileged --device /dev/fuse jobs: - 'swh-jobs-{name}' diff --git a/jobs/templates/debian/update-for-release.groovy.j2 b/jobs/templates/debian/update-for-release.groovy.j2 index fbdb1d9..7d2782c 100644 --- a/jobs/templates/debian/update-for-release.groovy.j2 +++ b/jobs/templates/debian/update-for-release.groovy.j2 @@ -1,157 +1,157 @@ def upstream_tag = params.GIT_TAG def version = upstream_tag.substring(1) def repo_name = '{{repo_name}}' -def python_module = repo_name.replace('-', '.') +def python_module = '{{python_module}}' def full_environ def debian_upstream_tag_exists = false pipeline { agent { label 'debian' } stages { stage('Checkout') { steps { cleanWs() checkout([ $class: 'GitSCM', branches: [[name: 'debian/unstable-swh'], [name: 'debian/*'], [name: 'pristine-tar']], userRemoteConfigs: [[ url: "https://forge.softwareheritage.org/source/${repo_name}.git", ]], extensions: [ [$class: 'RelativeTargetDirectory', relativeTargetDir: repo_name], ], poll: false, ]) dir (repo_name) { sh ''' git checkout -B pristine-tar origin/pristine-tar git checkout -B debian/upstream origin/debian/upstream git checkout -B debian/unstable-swh origin/debian/unstable-swh ''' script { def tag_exists_retval = sh( script: "git rev-parse --verify debian/upstream/${version}", returnStatus: true ) debian_upstream_tag_exists = (tag_exists_retval == 0) } } } } stage('Get author information from tag') { when { beforeAgent true expression { !debian_upstream_tag_exists } } steps { dir (repo_name) { // This script retrieves the author data for the tag that we're currently processing script { // Check if the tag is annotated or not def objecttype = sh( script: "git tag -l --format='%(*objecttype)' ${upstream_tag}", returnStdout: true, ).trim(); def tagger = !objecttype.isEmpty() ? 'tagger' : 'author'; def tagger_name = sh( script: "git tag -l --format='%(${tagger}name)' ${upstream_tag}", returnStdout: true, ).trim(); def tagger_email = sh( script: "git tag -l --format='%(${tagger}email)' ${upstream_tag}", returnStdout: true, ).trim(); tagger_email = tagger_email.substring(1, tagger_email.length() - 1); def tagger_date = sh( script: "git tag -l --format='%(${tagger}date:iso)' ${upstream_tag}", returnStdout: true, ).trim(); def hostname = sh( script: "hostname --fqdn", returnStdout: true, ).trim(); def short_hostname = hostname - '.internal.softwareheritage.org'; full_environ = [ "DEBEMAIL=jenkins@${hostname}", "DEBFULLNAME=Software Heritage autobuilder (on ${short_hostname})", "UPSTREAM_TAG=${upstream_tag}", "UPSTREAM_TAGGER_NAME=${tagger_name}", "UPSTREAM_TAGGER_EMAIL=${tagger_email}", "UPSTREAM_TAGGER_DATE=${tagger_date}", "UPSTREAM_VERSION=${version}", "PYTHON_MODULE=${python_module}", ]; } } } } stage('gbp import-orig') { when { beforeAgent true expression { !debian_upstream_tag_exists } } steps { copyArtifacts( projectName: '/{{name}}/pypi-upload', parameters: 'GIT_TAG=' + params.GIT_TAG, ) withEnv(full_environ) { dir (repo_name) { sh ''' gbp import-orig --merge-mode=replace --no-symlink-orig -u ${UPSTREAM_VERSION} ../dist/${PYTHON_MODULE}-${UPSTREAM_VERSION}.tar.gz ''' } } } } stage('unstable-swh changelog') { when { beforeAgent true expression { !debian_upstream_tag_exists } } steps { withEnv(full_environ) { dir(repo_name) { sh ''' dch -v ${UPSTREAM_VERSION}-1~swh1 "New upstream release ${UPSTREAM_VERSION}\n - (tagged by ${UPSTREAM_TAGGER_NAME} <${UPSTREAM_TAGGER_EMAIL}> on ${UPSTREAM_TAGGER_DATE})" dch "Upstream changes:\n$(git tag -l --format='%(contents:subject)%0a%(contents:body)' ${UPSTREAM_TAG} | sed -E -e '/^$/d' -e 's/^ *(- *)?//' -e 's/^/ - /')" dch -D unstable-swh --force-distribution '' git add debian/changelog git commit --no-verify -m "Updated debian changelog for version ${UPSTREAM_VERSION}" git show gbp buildpackage --git-tag-only ''' } } } } stage('Upload changes') { when { beforeAgent true expression { !debian_upstream_tag_exists } } steps { dir (repo_name) { sshagent (credentials: ['jenkins-public-ci-ssh']) { script { def git_push = 'git push --follow-tags --all' if (params.DRY_RUN) { git_push += ' -n' } sh(script: git_push) } } } } } } }