diff --git a/jobs/swh/sandbox.yaml b/jobs/swh/sandbox.yaml index 4978902..a8c0972 100644 --- a/jobs/swh/sandbox.yaml +++ b/jobs/swh/sandbox.yaml @@ -1,11 +1,12 @@ - project: name: TSBX display-name: swh-sandbox pkg: sandbox jobs: - "{name}" - "{name}/incoming-tag" - "{name}/tests" - "{name}/tox" + - "{name}/pypi-upload" - "debian/packages/{name}" - "debian/packages/{name}/update-for-release" diff --git a/jobs/templates/incoming-tag.groovy b/jobs/templates/incoming-tag.groovy index 4001cdd..45b3f59 100644 --- a/jobs/templates/incoming-tag.groovy +++ b/jobs/templates/incoming-tag.groovy @@ -1,78 +1,77 @@ import jenkins.model.Jenkins @NonCPS def job_exists(name) {{ def instance = jenkins.model.Jenkins.instance return instance.getItemByFullName(name) != null }} pipeline {{ agent none stages {{ stage('Refresh tag list') {{ agent any steps {{ checkout([ $class: 'GitSCM', userRemoteConfigs: [[ url: 'https://forge.softwareheritage.org/source/{display-name}.git', ]], branches: [[ name: params.GIT_TAG, ]], browser: [ $class: 'Phabricator', repo: '{display-name}', repoUrl: 'https://forge.softwareheritage.org/', ], ]) }} }} stage('Build and upload PyPI package') {{ when {{ expression {{ params.GIT_TAG =~ /^v\d+(.\d+)+$/ }} expression {{ job_exists('/{name}/pypi') }} }} steps {{ build( - job: '/{name}/pypi', + job: '/{name}/pypi-upload', parameters: [ string(name: 'GIT_TAG', value: params.GIT_TAG), ], - wait: false, ) }} }} stage('Debian packaging for new release') {{ when {{ expression {{ params.GIT_TAG =~ /^v\d+(.\d+)+$/ }} expression {{ job_exists('/debian/packages/{name}/update-for-release') }} }} steps {{ build( job: '/debian/packages/{name}/update-for-release', parameters: [ string(name: 'GIT_TAG', value: params.GIT_TAG), ], wait: false, ) }} }} stage('Debian automatic build') {{ when {{ expression {{ params.GIT_TAG =~ /^debian\/.*$/ }} expression {{ job_exists('/debian/packages/{name}/build') }} }} steps {{ build( job: '/debian/packages/{name}/build', parameters: [ string(name: 'GIT_TAG', value: params.GIT_TAG), booleanParam(name: 'UPLOAD', value: true), ], wait: false, ) }} }} }} }} diff --git a/jobs/templates/swh-pipeline.yaml b/jobs/templates/swh-pipeline.yaml index 9fff44a..56e5147 100644 --- a/jobs/templates/swh-pipeline.yaml +++ b/jobs/templates/swh-pipeline.yaml @@ -1,178 +1,130 @@ - job-template: name: "{name}/tests" display-name: "master branch" project-type: pipeline triggers: - timed: '@daily' sandbox: true auth-token: 'ph4br1cat0r' properties: - build-discarder: num-to-keep: 20 parameters: - git-parameter: name: REVISION type: PT_REVISION defaultValue: master description: Git revision to build. - string: name: PHID description: PHID of the Phabricator target object on which results will be reported. - - string: - name: PYPI - description: | - PYPI registry to publish to when built revision is a tag. There - must exists a usernamePassword credential object with that name. - The pypi JSON API endpoint is built as https://$PYPI/project//json - default: test.pypi.org - - string: - name: PYPI_UPLOAD - description: | - PYPI upload base host. If unset, defaults to $PYPI. The upload endpoint is built as - https://$PYPI_UPLOAD/legagy/ - - bool: - name: FORCE_UPLOAD - default: false - description: | - Force uploading python packages on the chosen PYPI registry. dsl: | pipeline {{ agent {{ label 'swh-tox' }} environment {{ PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' }} stages {{ stage('Checkout') {{ steps {{ withCredentials([ string(credentialsId: 'swh-public-ci', variable: 'PHAB_CONDUIT_TOKEN')]) {{ sh ''' if [ -n "$PHID" ]; then python3 -m pyarcanist send-message work $PHID fi ''' }} checkout([$class: 'GitSCM', branches: [[name: "${{params.REVISION}}"]], doGenerateSubmoduleConfigurations: false, extensions: [], gitTool: 'Default', submoduleCfg: [], userRemoteConfigs: [[url: 'https://forge.softwareheritage.org/source/{display-name}.git']] ]) }} }} stage('Static analysis') {{ steps {{ echo 'flake8' sh '''python3 -m detox -e flake8''' echo 'radon'; sh ''' mkdir -p reports python3 -m radon raw --json swh/ > reports/raw_report.json python3 -m radon cc --json swh/ > reports/cc_report.json python3 -m radon mi --json swh/ > reports/mi_report.json python3 -m radon hal --json swh/ > reports/hal_report.json python3 -m radon cc --xml swh/ > reports/cc_report.xml ''' }} }} // static analysis stage('Unit tests') {{ options {{ timeout(time: 5, unit: 'MINUTES') }} steps {{ sh ''' python3 -m tox -e py3 -- \ --cov-report=xml \ --junit-xml=test-results.xml ''' }} post {{ always {{ step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: 'coverage.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 10, onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false]) // JUnit report junit allowEmptyResults: true, testResults: 'test-results.xml' // disabled for now, requires the plugin Warning v5 (still in RC) //recordIssues enabledForFailure: true, // tools: [[pattern: '**/reports/cc_report.xml', tool: [$class: 'Ccm']]] }} }} // post }} // unit tests - - stage('Build') {{ - steps {{ - sh 'python3 setup.py sdist bdist_wheel' - }} - }} - - stage('Publish') {{ - when {{ anyOf {{ - expression {{ return params.FORCE_UPLOAD }} - expression {{ - GITTAG=sh(returnStdout: true, - script:'git describe --exact-match 2>/dev/null || true').trim() - LASTV=sh(returnStdout: true, - script:'curl -s https://${{PYPI}}/pypi/`python setup.py --name`/json | jq -r .info.version || true').trim() - return ! (GITTAG in ['', 'v'+LASTV]) - }} - }} - }} - steps {{ - withCredentials([ - usernamePassword(credentialsId: "${{params.PYPI}}", - usernameVariable: 'TWINE_USERNAME', - passwordVariable: 'TWINE_PASSWORD')]) {{ - sh ''' - if [ -z "$PYPI_UPLOAD" ]; then PYPI_UPLOAD=$PYPI; fi - python3 -m twine upload --repository-url https://${{PYPI_UPLOAD}}/legacy/ dist/* - ''' - }} - }} - }} }} // stages post {{ always {{ // Archive a few report files archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/*,*.xml,tox*.ini', fingerprint: true withCredentials([ string(credentialsId: 'swh-public-ci', variable: 'PHAB_CONDUIT_TOKEN')]) {{ withEnv(["JOBSTATUS=${{currentBuild.currentResult}}"]) {{ sh ''' if [ "$JOBSTATUS" = "SUCCESS" ]; then MSGTYPE=pass else MSGTYPE=fail fi echo "Current job status is $JOBSTATUS -> $MGSTYPE" if [ -n "$PHID" ]; then python3 -m pyarcanist send-message $MSGTYPE $PHID fi ''' }} // withEnv }} // withCredentials }} // always }} // post }} // pipeline diff --git a/jobs/templates/swh-pypi.groovy b/jobs/templates/swh-pypi.groovy new file mode 100644 index 0000000..1423d46 --- /dev/null +++ b/jobs/templates/swh-pypi.groovy @@ -0,0 +1,64 @@ +pipeline {{ + agent {{ label 'swh-tox' }} + + stages {{ + stage('Run tests') {{ + agent none + steps {{ + build( + job: '/{name}/tests', + parameters: [ + string(name: 'REVISION', value: params.GIT_TAG), + ], + propagate: !params.IGNORE_TESTS, + ) + }} + }} + + stage('Checkout') {{ + steps {{ + checkout([$class: 'GitSCM', + branches: [[name: params.GIT_TAG]], + doGenerateSubmoduleConfigurations: false, + extensions: [], + gitTool: 'Default', + submoduleCfg: [], + userRemoteConfigs: [[url: 'https://forge.softwareheritage.org/source/{display-name}.git']] + ]) + }} + }} + + stage('Build') {{ + steps {{ + sh 'python3 setup.py sdist bdist_wheel' + archiveArtifacts allowEmptyArchive: true, + artifacts: 'dist/*', + fingerprint: true + }} + }} + + stage('Publish') {{ + when {{ + anyOf {{ + expression {{ return params.FORCE_UPLOAD }} + expression {{ + LASTV=sh(returnStdout: true, + script:'curl -s https://${{PYPI}}/pypi/`python setup.py --name`/json | jq -r .info.version || true').trim() + return 'v'+LASTV != params.GIT_TAG + }} + }} + }} + steps {{ + withCredentials([ + usernamePassword(credentialsId: "${{params.PYPI}}", + usernameVariable: 'TWINE_USERNAME', + passwordVariable: 'TWINE_PASSWORD')]) {{ + sh ''' + if [ -z "$PYPI_UPLOAD" ]; then PYPI_UPLOAD=$PYPI; fi + python3 -m twine upload --repository-url https://${{PYPI_UPLOAD}}/legacy/ dist/* + ''' + }} + }} + }} + }} +}} diff --git a/jobs/templates/swh-pypi.yaml b/jobs/templates/swh-pypi.yaml new file mode 100644 index 0000000..a718eeb --- /dev/null +++ b/jobs/templates/swh-pypi.yaml @@ -0,0 +1,40 @@ +- job-template: + name: "{name}/pypi-upload" + display-name: "Upload to PyPI" + project-type: pipeline + sandbox: true + properties: + - build-discarder: + num-to-keep: 20 + parameters: + - git-parameter: + name: GIT_TAG + description: git tag to process + type: PT_TAG + sortMode: DESCENDING_SMART + selectedValue: TOP + - string: + name: PYPI + description: | + PYPI registry to publish to when built revision is a tag. There + must exists a usernamePassword credential object with that name. + The pypi JSON API endpoint is built as https://$PYPI/project//json + default: test.pypi.org + - string: + name: PYPI_UPLOAD + description: | + PYPI upload base host. If unset, defaults to $PYPI. The upload endpoint is built as + https://$PYPI_UPLOAD/legacy/ + - bool: + name: FORCE_UPLOAD + default: false + description: | + Force uploading python packages on the chosen PYPI registry, even if + the package already exists. + - bool: + name: IGNORE_TESTS + default: false + description: | + Proceed even if the tests are failing on the repository. + dsl: + !include-raw: swh-pypi.groovy