diff --git a/jobs/swh-packages.yaml b/jobs/swh-packages.yaml index eab7767..1785b01 100644 --- a/jobs/swh-packages.yaml +++ b/jobs/swh-packages.yaml @@ -1,157 +1,155 @@ - job-group: name: 'swh-jobs-{name}' jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tox" - "debian/packages/{name}" - "debian/packages/{name}/update-for-release" - "debian/packages/{name}/automatic-backport" - "debian/packages/{name}/gbp-buildpackage" - project: name: DCORE display-name: swh-core pkg: core jobs: - 'swh-jobs-{name}' - project: name: DDEP display-name: swh-deposit pkg: deposit jobs: - 'swh-jobs-{name}' - project: name: DGRPH display-name: swh-graph pkg: graph jobs: - 'swh-jobs-{name}' - project: name: DCIDX display-name: swh-indexer pkg: indexer jobs: - 'swh-jobs-{name}' - project: name: DICP display-name: swh-icinga-plugins pkg: icinga-plugins jobs: - 'swh-jobs-{name}' - project: name: DJNL display-name: swh-journal docker-image: swh-kafka pkg: journal jobs: - 'swh-jobs-{name}' - project: name: DLS display-name: swh-lister pkg: lister jobs: - 'swh-jobs-{name}' - project: name: DLDBASE display-name: swh-loader-core pkg: loader.core jobs: - 'swh-jobs-{name}' - project: name: DLDG display-name: swh-loader-git pkg: loader.git jobs: - 'swh-jobs-{name}' - project: name: DLDHG display-name: swh-loader-mercurial pkg: loader.mercurial jobs: - 'swh-jobs-{name}' - project: name: DLDSVN display-name: swh-loader-svn pkg: loader.svn jobs: - 'swh-jobs-{name}' - project: name: DMOD display-name: swh-model pkg: model jobs: - 'swh-jobs-{name}' - project: name: DOBJS display-name: swh-objstorage pkg: objstorage deb-extra-repositories: ceph jobs: - 'swh-jobs-{name}' - project: name: DSEA display-name: swh-search pkg: search jobs: - 'swh-jobs-{name}' - project: name: TSBX display-name: swh-sandbox pkg: sandbox incoming-tag-auto-pypi-host: test.pypi.org jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tox" - project: name: DSCH display-name: swh-scheduler pkg: scheduler jobs: - 'swh-jobs-{name}' - project: name: DSTO display-name: swh-storage pkg: storage jobs: - 'swh-jobs-{name}' - - '{name}/tox': - pytest_params: -m cassandra -n 3 - project: name: DVAU display-name: swh-vault pkg: vault jobs: - 'swh-jobs-{name}' - project: name: DWAPPS display-name: swh-web pkg: web jobs: - 'swh-jobs-{name}' - '{name}/cypress' - '{name}/cypress-diff' diff --git a/jobs/templates/swh-pipeline.groovy b/jobs/templates/swh-pipeline.groovy index 8ef64ee..c3c2955 100644 --- a/jobs/templates/swh-pipeline.groovy +++ b/jobs/templates/swh-pipeline.groovy @@ -1,124 +1,124 @@ pipeline {{ agent {{ label '{docker-image}' }} environment {{ PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' }} stages {{ stage('Checkout') {{ steps {{ withCredentials([ string(credentialsId: 'swh-public-ci', variable: 'PHAB_CONDUIT_TOKEN')]) {{ sh ''' if [ -n "$PHID" ]; then echo "{{ \\\"buildTargetPHID\\\": \\\"$PHID\\\", \\\"artifactKey\\\": \\\"link.jenkins\\\", \\\"artifactType\\\": \\\"uri\\\", \\\"artifactData\\\": {{ \\\"uri\\\": \\\"$BUILD_URL\\\", \\\"name\\\": \\\"Jenkins\\\", \\\"ui.external\\\": true }} }}" | arc call-conduit --conduit-uri $PHAB_CONDUIT_URL --conduit-token $PHAB_CONDUIT_TOKEN harbormaster.createartifact python3 -m pyarcanist send-message work $PHID fi ''' }} checkout([$class: 'GitSCM', branches: [[name: "${{params.REVISION}}"]], doGenerateSubmoduleConfigurations: false, extensions: [], gitTool: 'Default', submoduleCfg: [], userRemoteConfigs: [[url: 'https://forge.softwareheritage.org/source/{display-name}.git']], browser: [$class: 'Phabricator', repoUrl: 'https://forge.softwareheritage.org', repo: '{name}'] ]) }} }} stage('Static analysis') {{ steps {{ echo 'flake8' sh '''python3 -m tox -e flake8''' echo 'mypy' sh '''if python3 -m tox -a | grep -qx mypy ; then python3 -m tox -e mypy ; fi''' echo 'radon'; sh ''' mkdir -p reports python3 -m radon raw --json swh/ > reports/raw_report.json python3 -m radon cc --json swh/ > reports/cc_report.json python3 -m radon mi --json swh/ > reports/mi_report.json python3 -m radon hal --json swh/ > reports/hal_report.json python3 -m radon cc --xml swh/ > reports/cc_report.xml ''' }} }} // static analysis stage('Unit tests') {{ options {{ timeout(time: 20, unit: 'MINUTES') }} steps {{ sh ''' - python3 -m tox -e py3 -- \ + python3 -m tox -e $TOX_ENVIRONMENT -- \ --cov-report=xml \ --junit-xml=test-results.xml ''' }} post {{ always {{ step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: 'coverage.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 10, onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false]) // JUnit report junit allowEmptyResults: true, testResults: 'test-results.xml' // disabled for now, requires the plugin Warning v5 (still in RC) //recordIssues enabledForFailure: true, // tools: [[pattern: '**/reports/cc_report.xml', tool: [$class: 'Ccm']]] }} }} // post }} // unit tests }} // stages post {{ always {{ // Archive a few report files archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/*,*.xml,tox*.ini', fingerprint: true withCredentials([ string(credentialsId: 'swh-public-ci', variable: 'PHAB_CONDUIT_TOKEN')]) {{ withEnv(["JOBSTATUS=${{currentBuild.currentResult}}"]) {{ sh ''' if [ "$JOBSTATUS" = "SUCCESS" ]; then MSGTYPE=pass else MSGTYPE=fail fi echo "Current job status is $JOBSTATUS -> $MGSTYPE" if [ -n "$PHID" ]; then python3 -m pyarcanist send-message $MSGTYPE $PHID fi ''' }} // withEnv }} // withCredentials }} // always }} // post }} // pipeline diff --git a/jobs/templates/swh-pipeline.yaml b/jobs/templates/swh-pipeline.yaml index 75a5cf1..5b4f098 100644 --- a/jobs/templates/swh-pipeline.yaml +++ b/jobs/templates/swh-pipeline.yaml @@ -1,25 +1,30 @@ - job-template: name: "{name}/tests" display-name: "master branch" project-type: pipeline docker-image: swh-tox + default-tox-environment: py3 triggers: - timed: '@daily' sandbox: true auth-token: 'ph4br1cat0r' properties: - build-discarder: num-to-keep: 20 parameters: - git-parameter: name: REVISION type: PT_REVISION defaultValue: master description: Git revision to build. - string: name: PHID description: PHID of the Phabricator target object on which results will be reported. + - string: + name: TOX_ENVIRONMENT + description: tox environment to use for the main tox run + default: "{default-tox-environment}" dsl: !include-raw: swh-pipeline.groovy diff --git a/jobs/templates/swh-tox.yaml b/jobs/templates/swh-tox.yaml index f645972..0d779fc 100644 --- a/jobs/templates/swh-tox.yaml +++ b/jobs/templates/swh-tox.yaml @@ -1,56 +1,57 @@ - job-template: name: '{name}/tox' display-name: 'Phab. Diff' docker-image: swh-tox node: "{docker-image}" + default-tox-environment: py3 auth-token: ph4br1cat0r parameters: - string: name: REPO description: PHID of the Diffusion repository - string: name: PHID description: PHID of the Target object - string: name: DIFF_ID description: ID of the Diff patch to apply, if any - string: - name: pytest_params - description: extra parameters to pass to pytest - default: "" + name: TOX_ENVIRONMENT + description: tox environment to use for the main tox run + default: "{default-tox-environment}" scm: - git: url: https://forge.softwareheritage.org/source/{display-name}.git branches: - origin/master wrappers: - phabricator-differential: apply-to-master: true patch-with-force-flag: true builders: - shell: | echo "Run tox for:" echo " REPO=$REPO" echo " PHID=$PHID" echo " DIFF_ID=$DIFF_ID" - shell: | - if (python3 -m tox -a | grep -q -e '^flake8-phab$' ); then + if (python3 -m tox -a | grep -qx flake8-phab ); then python3 -m tox -e flake8-phab else python3 -m tox -e flake8 fi - - shell: python3 -m tox --result-json tox-results.json -- $pytest_params + - shell: python3 -m tox --result-json tox-results.json -e $TOX_ENVIRONMENT publishers: - phabricator: uberalls-enabled: false comment-on-success: true comment-with-console-link-on-failure: true process-lint: true lint-file: .phabricator-lint lint-file-size: 1000000 - archive: artifacts: "*.json,.coverage*,.phabricator*"