diff --git a/jobs/swh-packages.yaml b/jobs/swh-packages.yaml index 5c34747..ef9e342 100644 --- a/jobs/swh-packages.yaml +++ b/jobs/swh-packages.yaml @@ -1,311 +1,312 @@ - job-group: name: "swh-jobs-{name}" jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tests-on-diff" - "debian/packages/{name}" - "debian/packages/{name}/update-for-release" - "debian/packages/{name}/automatic-backport" - "debian/packages/{name}/gbp-buildpackage" - project: name: DAUTH display-name: swh-auth repo_name: swh-auth pkg: auth python_module: swh.auth jobs: - "swh-jobs-{name}" - project: name: DCORE display-name: swh-core repo_name: swh-core pkg: core python_module: swh.core jobs: - "swh-jobs-{name}" - project: name: DCNT display-name: swh-counters repo_name: swh-counters pkg: counters python_module: swh.counters jobs: - "swh-jobs-{name}" - project: name: DDATASET display-name: swh-dataset repo_name: swh-dataset pkg: dataset python_module: swh.dataset jobs: - "swh-jobs-{name}" - project: name: DDEP display-name: swh-deposit repo_name: swh-deposit pkg: deposit python_module: swh.deposit jobs: - "swh-jobs-{name}" - project: name: DGRPH display-name: swh-graph repo_name: swh-graph pkg: graph python_module: swh.graph jobs: - "swh-jobs-{name}" - project: name: DCIDX display-name: swh-indexer repo_name: swh-indexer pkg: indexer python_module: swh.indexer jobs: - "swh-jobs-{name}" - project: name: DICP display-name: swh-icinga-plugins repo_name: swh-icinga-plugins pkg: icinga-plugins python_module: swh.icinga_plugins jobs: - "swh-jobs-{name}" - project: name: DJNL display-name: swh-journal repo_name: swh-journal docker_image: kafka pkg: journal python_module: swh.journal jobs: - "swh-jobs-{name}" - project: name: DLS display-name: swh-lister repo_name: swh-lister pkg: lister python_module: swh.lister jobs: - "swh-jobs-{name}" - project: name: DLDBZR display-name: swh-loader-bzr repo_name: swh-loader-bzr pkg: loader.bzr python_module: swh.loader.bzr jobs: - "swh-jobs-{name}" - project: name: DLDBASE display-name: swh-loader-core repo_name: swh-loader-core pkg: loader.core python_module: swh.loader.core jobs: - "swh-jobs-{name}" - project: name: DLDCVS display-name: swh-loader-cvs repo_name: swh-loader-cvs pkg: loader.cvs python_module: swh.loader.cvs jobs: - "swh-jobs-{name}" - project: name: DLDG display-name: swh-loader-git repo_name: swh-loader-git pkg: loader.git python_module: swh.loader.git jobs: - "swh-jobs-{name}" - project: name: DLDHG display-name: swh-loader-mercurial repo_name: swh-loader-mercurial pkg: loader.mercurial python_module: swh.loader.mercurial jobs: - "swh-jobs-{name}" - project: name: DLDSVN display-name: swh-loader-svn repo_name: swh-loader-svn pkg: loader.svn python_module: swh.loader.svn jobs: - "swh-jobs-{name}" - project: name: DMOD display-name: swh-model repo_name: swh-model pkg: model python_module: swh.model jobs: - "swh-jobs-{name}" - project: name: DMFCD display-name: swh-clearlydefined repo_name: swh-clearlydefined pkg: clearlydefined python_module: swh.clearlydefined jobs: - "swh-jobs-{name}" - project: name: DOBJS display-name: swh-objstorage repo_name: swh-objstorage pkg: objstorage deb-extra-repositories: ceph python_module: swh.objstorage jobs: - "swh-jobs-{name}" - project: name: DOBJSRPL display-name: swh-objstorage-replayer repo_name: swh-objstorage-replayer pkg: objstorage.replayer python_module: swh.objstorage.replayer jobs: - "swh-jobs-{name}" - project: name: DOPH display-name: swh-perfecthash repo_name: swh-perfecthash + include_bdist: false pkg: perfecthash python_module: swh.perfecthash jobs: - "swh-jobs-{name}" - project: name: DPROV display-name: swh-provenance repo_name: swh-provenance pkg: provenance python_module: swh.provenance jobs: - "swh-jobs-{name}" - project: name: DSEA display-name: swh-search repo_name: swh-search pkg: search deb-extra-repositories: elasticsearch python_module: swh.search jobs: - "swh-jobs-{name}" - project: name: TSBX display-name: swh-sandbox repo_name: swh-sandbox pkg: sandbox incoming_tag_auto_pypi_host: test.pypi.org python_module: swh.sandbox jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tests-on-diff" - project: name: DTSCN display-name: swh-scanner repo_name: swh-scanner pkg: scanner python_module: swh.scanner jobs: - "swh-jobs-{name}" - project: name: DSCH display-name: swh-scheduler repo_name: swh-scheduler pkg: scheduler python_module: swh.scheduler jobs: - "swh-jobs-{name}" - project: name: DSTO display-name: swh-storage repo_name: swh-storage pkg: storage deb-extra-repositories: cassandra python_module: swh.storage jobs: - "swh-jobs-{name}" - project: name: DVAU display-name: swh-vault repo_name: swh-vault pkg: vault python_module: swh.vault jobs: - "swh-jobs-{name}" - project: name: DWAPPS display-name: swh-web repo_name: swh-web pkg: web python_module: swh.web do_cypress: true jobs: - "swh-jobs-{name}" - project: name: DWCLI display-name: swh-web-client repo_name: swh-web-client pkg: web-client python_module: swh.web.client jobs: - "swh-jobs-{name}" - project: name: DFUSE display-name: swh-fuse repo_name: swh-fuse pkg: fuse python_module: swh.fuse docker_options: --privileged --device /dev/fuse jobs: - "swh-jobs-{name}" - project: name: DTPL display-name: swh-py-template repo_name: swh-py-template pkg: py-template python_module: swh.foo jobs: - "swh-jobs-{name}" diff --git a/jobs/templates/swh-pypi.groovy.j2 b/jobs/templates/swh-pypi.groovy.j2 index 98def15..a17158c 100644 --- a/jobs/templates/swh-pypi.groovy.j2 +++ b/jobs/templates/swh-pypi.groovy.j2 @@ -1,101 +1,107 @@ def module_name = '{{repo_name}}'.replace('-', '.') def PYPI_UPLOAD_HOST switch (params.PYPI_HOST) { case 'pypi.org': PYPI_UPLOAD_HOST = 'upload.pypi.org' break default: PYPI_UPLOAD_HOST = params.PYPI_HOST break } +def BDIST_WHEEL = 'bdist_wheel' + +if (!params.INCLUDE_BDIST) { + BDIST_WHEEL = '' +} + pipeline { {% filter indent(width=2) %} {%- include 'includes/agent-docker.groovy.j2' -%} {% endfilter %} stages { stage('Run tests') { when { expression { return !params.SKIP_TESTS } beforeAgent true } agent none steps { build( job: '/{{name}}/tests', parameters: [ string(name: 'REVISION', value: params.GIT_TAG), ], propagate: !params.IGNORE_TESTS, ) } } stage('Checkout') { steps { checkout([$class: 'GitSCM', branches: [[name: params.GIT_TAG]], doGenerateSubmoduleConfigurations: false, extensions: [], gitTool: 'Default', submoduleCfg: [], userRemoteConfigs: [[url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git']] ]) sh '''rm -rf dist/''' } } stage('Build') { steps { sh ''' # Build javascript assets if [ -f yarn.lock ]; then yarn install --frozen-lockfile yarn build fi # Build java assets if [ -d java ]; then for pom in $( find java/ -name pom.xml ) ; do mvn -f $pom compile assembly:single done fi - python3 setup.py sdist bdist_wheel + python3 setup.py sdist ${BDIST_WHEEL} ''' archiveArtifacts allowEmptyArchive: true, artifacts: 'dist/*', fingerprint: true } } stage('Publish') { when { anyOf { expression { return params.FORCE_UPLOAD } expression { LASTV=sh(returnStdout: true, script:"curl -s https://${params.PYPI_HOST}/pypi/${module_name}/json | jq -r .info.version || true").trim() return 'v'+LASTV != params.GIT_TAG } } } steps { withCredentials([ usernamePassword(credentialsId: PYPI_UPLOAD_HOST, usernameVariable: 'TWINE_USERNAME', passwordVariable: 'TWINE_PASSWORD')]) { sh "python3 -m twine upload --repository-url https://${PYPI_UPLOAD_HOST}/legacy/ dist/*" } } } } post { cleanup { cleanWs() } } } diff --git a/jobs/templates/swh-pypi.yaml b/jobs/templates/swh-pypi.yaml index d7c7e71..9d4cc07 100644 --- a/jobs/templates/swh-pypi.yaml +++ b/jobs/templates/swh-pypi.yaml @@ -1,46 +1,52 @@ - job-template: name: "{name}/pypi-upload" display-name: "Upload to PyPI" docker_image: tox + include_bdist: true project-type: pipeline sandbox: true properties: - build-discarder: artifact-num-to-keep: 10 - copyartifact: projects: '*' parameters: - git-parameter: name: GIT_TAG description: git tag to process type: PT_TAG sortMode: DESCENDING_SMART selectedValue: TOP - choice: name: PYPI_HOST description: | PYPI registry to publish to when built revision is a tag. There must exists a usernamePassword credential object with that name. The pypi JSON API endpoint is built as https://$PYPI_HOST/project//json choices: - test.pypi.org - pypi.org default: test.pypi.org - bool: name: FORCE_UPLOAD default: false description: | Force uploading python packages on the chosen PYPI registry, even if the package already exists. - bool: name: SKIP_TESTS default: false description: | Do not run tests on the repository. - bool: name: IGNORE_TESTS default: false description: | Proceed even if the tests are failing on the repository. + - bool: + name: INCLUDE_BDIST + default: '{include_bdist}' + description: | + Include a binary distribution in the PyPI upload. dsl: !include-jinja2: swh-pypi.groovy.j2