diff --git a/jobs/templates/includes/agent-docker.groovy.j2 b/jobs/templates/includes/agent-docker.groovy.j2 new file mode 100644 index 0000000..ea6b5b1 --- /dev/null +++ b/jobs/templates/includes/agent-docker.groovy.j2 @@ -0,0 +1,8 @@ +agent { + docker { + reuseNode true + image 'swh-jenkins/{{docker_image}}' + args '--tmpfs /tmp:exec --mount type=volume,src=shared-jenkins-cachedir,dst=/home/jenkins/.cache' + } +} + diff --git a/jobs/templates/swh-pipeline.groovy.j2 b/jobs/templates/swh-pipeline.groovy.j2 index e967f91..409f16e 100644 --- a/jobs/templates/swh-pipeline.groovy.j2 +++ b/jobs/templates/swh-pipeline.groovy.j2 @@ -1,116 +1,113 @@ pipeline { - agent { - docker { - image 'swh-jenkins/{{docker_image}}' - args '--tmpfs /tmp:exec --mount type=volume,src=shared-jenkins-cachedir,dst=/home/jenkins/.cache' - } - } + {% filter indent(width=2) %} + {%- include 'includes/agent-docker.groovy.j2' -%} + {% endfilter %} environment { PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' } stages { stage('Checkout') { steps { {% filter indent(width=8) %} {%- include 'includes/create-phabricator-artifacts.groovy.j2' -%} {% endfilter %} checkout([ $class: 'GitSCM', branches: [[name: "${params.REVISION}"]], doGenerateSubmoduleConfigurations: false, extensions: [], gitTool: 'Default', submoduleCfg: [], userRemoteConfigs: [ [url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git'], ], browser: [$class: 'Phabricator', repoUrl: 'https://forge.softwareheritage.org', repo: '{{name}}'] ]) } } {%- if phabricator_diff %} {% filter indent(width=4) %} {%- include 'includes/stage-apply-phabricator-diff.groovy.j2' -%} {% endfilter %} {%- endif %} stage('Cleanup') { steps { {% filter indent(width=8) %} {%- include 'includes/cleanup-workspace.groovy.j2' -%} {% endfilter %} } } stage('Static analysis') { steps { echo 'flake8' sh '''python3 -m tox -e flake8''' echo 'mypy' sh '''if python3 -m tox -a 2>/dev/null | grep -qx mypy ; then python3 -m tox -e mypy ; fi''' echo 'radon'; sh ''' mkdir -p reports python3 -m radon raw --json swh/ > reports/raw_report.json python3 -m radon cc --json swh/ > reports/cc_report.json python3 -m radon mi --json swh/ > reports/mi_report.json python3 -m radon hal --json swh/ > reports/hal_report.json python3 -m radon cc --xml swh/ > reports/cc_report.xml ''' } post { always { // Warnings NG recordIssues( enabledForFailure: true, tools: [ ccm(pattern: '**/reports/cc_report.xml'), ], ) } } } // static analysis stage('Tests') { parallel { {% filter indent(width=8) %} {%- include 'includes/stage-python-tests.groovy.j2' -%} {% endfilter %} {%- if do_cypress %} {% filter indent(width=8) %} {%- include 'includes/stage-cypress-tests.groovy.j2' -%} {% endfilter %} {%- endif %} } } } // stages post { always { // Archive a few report files archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/*,*.xml,tox*.ini', fingerprint: true step([$class: 'PhabricatorNotifier', commentOnSuccess: true, commentWithConsoleLinkOnFailure: true, commentFile: '.phabricator-comment', commentSize: '1000000', preserveFormatting: true, processLint: true, lintFile: '.phabricator-lint', lintFileSize: '1000000', ]) {% filter indent(width=6) %} {%- include 'includes/cleanup-workspace.groovy.j2' -%} {% endfilter %} } // always } // post } // pipeline diff --git a/jobs/templates/swh-pypi.groovy.j2 b/jobs/templates/swh-pypi.groovy.j2 index 8eec948..282f910 100644 --- a/jobs/templates/swh-pypi.groovy.j2 +++ b/jobs/templates/swh-pypi.groovy.j2 @@ -1,115 +1,112 @@ def module_name = '{{repo_name}}'.replace('-', '.') def PYPI_UPLOAD_HOST switch (params.PYPI_HOST) { case 'pypi.org': PYPI_UPLOAD_HOST = 'upload.pypi.org' break default: PYPI_UPLOAD_HOST = params.PYPI_HOST break } pipeline { - agent { - docker { - image 'swh-jenkins/{{docker_image}}' - args '--tmpfs /tmp:exec --mount type=volume,src=shared-jenkins-cachedir,dst=/home/jenkins/.cache' - } - } + {% filter indent(width=2) %} + {%- include 'includes/agent-docker.groovy.j2' -%} + {% endfilter %} stages { stage('Run tests') { when { expression { return !params.SKIP_TESTS } beforeAgent true } agent none steps { build( job: '/{{name}}/tests', parameters: [ string(name: 'REVISION', value: params.GIT_TAG), ], propagate: !params.IGNORE_TESTS, ) } } stage('Run cypress tests') { when { expression { '{{repo_name}}' == 'swh-web' && !params.SKIP_TESTS } beforeAgent true } agent none steps { build( job: '/{{name}}/cypress', parameters: [ string(name: 'REVISION', value: params.GIT_TAG), ], propagate: !params.IGNORE_TESTS, ) } } stage('Checkout') { steps { checkout([$class: 'GitSCM', branches: [[name: params.GIT_TAG]], doGenerateSubmoduleConfigurations: false, extensions: [], gitTool: 'Default', submoduleCfg: [], userRemoteConfigs: [[url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git']] ]) } } stage('Build') { steps { sh ''' # Build javascript assets if [ -f yarn.lock ]; then yarn install --frozen-lockfile yarn build fi # Build java assets if [ -d java ]; then for pom in $( find java/ -name pom.xml ) ; do mvn -f $pom compile assembly:single done fi python3 setup.py sdist bdist_wheel ''' archiveArtifacts allowEmptyArchive: true, artifacts: 'dist/*', fingerprint: true } } stage('Publish') { when { anyOf { expression { return params.FORCE_UPLOAD } expression { LASTV=sh(returnStdout: true, script:"curl -s https://${params.PYPI_HOST}/pypi/${module_name}/json | jq -r .info.version || true").trim() return 'v'+LASTV != params.GIT_TAG } } } steps { withCredentials([ usernamePassword(credentialsId: PYPI_UPLOAD_HOST, usernameVariable: 'TWINE_USERNAME', passwordVariable: 'TWINE_PASSWORD')]) { sh "python3 -m twine upload --repository-url https://${PYPI_UPLOAD_HOST}/legacy/ dist/*" } } } } }