diff --git a/jobs/swh-packages.yaml b/jobs/swh-packages.yaml index 05537e2..ff551ab 100644 --- a/jobs/swh-packages.yaml +++ b/jobs/swh-packages.yaml @@ -1,171 +1,171 @@ - job-group: name: 'swh-jobs-{name}' jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tox" - "debian/packages/{name}" - "debian/packages/{name}/update-for-release" - "debian/packages/{name}/automatic-backport" - "debian/packages/{name}/gbp-buildpackage" - project: name: DCORE display-name: swh-core pkg: core jobs: - 'swh-jobs-{name}' - project: name: DDEP display-name: swh-deposit pkg: deposit jobs: - 'swh-jobs-{name}' - project: name: DGRPH display-name: swh-graph pkg: graph jobs: - 'swh-jobs-{name}' - project: name: DCIDX display-name: swh-indexer pkg: indexer jobs: - 'swh-jobs-{name}' - project: name: DICP display-name: swh-icinga-plugins pkg: icinga-plugins jobs: - 'swh-jobs-{name}' - project: name: DJNL display-name: swh-journal - docker-image: kafka + docker_image: kafka pkg: journal jobs: - 'swh-jobs-{name}' - project: name: DLS display-name: swh-lister pkg: lister jobs: - 'swh-jobs-{name}' - project: name: DLDBASE display-name: swh-loader-core pkg: loader.core jobs: - 'swh-jobs-{name}' - project: name: DLDG display-name: swh-loader-git pkg: loader.git jobs: - 'swh-jobs-{name}' - project: name: DLDHG display-name: swh-loader-mercurial pkg: loader.mercurial jobs: - 'swh-jobs-{name}' - project: name: DLDSVN display-name: swh-loader-svn pkg: loader.svn jobs: - 'swh-jobs-{name}' - project: name: DMOD display-name: swh-model pkg: model jobs: - 'swh-jobs-{name}' - project: name: DOBJS display-name: swh-objstorage pkg: objstorage deb-extra-repositories: ceph jobs: - 'swh-jobs-{name}' - project: name: DSEA display-name: swh-search pkg: search deb-extra-repositories: elasticsearch jobs: - 'swh-jobs-{name}' - project: name: TSBX display-name: swh-sandbox pkg: sandbox incoming-tag-auto-pypi-host: test.pypi.org jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tox" - project: name: DTSCN display-name: swh-scanner pkg: scanner jobs: - 'swh-jobs-{name}' - project: name: DSCH display-name: swh-scheduler pkg: scheduler jobs: - 'swh-jobs-{name}' - project: name: DSTO display-name: swh-storage pkg: storage deb-extra-repositories: cassandra jobs: - 'swh-jobs-{name}' - project: name: DVAU display-name: swh-vault pkg: vault jobs: - 'swh-jobs-{name}' - project: name: DWAPPS display-name: swh-web pkg: web jobs: - 'swh-jobs-{name}' - '{name}/cypress' - '{name}/cypress-diff' - project: name: DWCLI display-name: swh-web-client pkg: web-client jobs: - 'swh-jobs-{name}' diff --git a/jobs/templates/swh-cypress.groovy b/jobs/templates/swh-cypress.groovy index bbae242..f84c1ac 100644 --- a/jobs/templates/swh-cypress.groovy +++ b/jobs/templates/swh-cypress.groovy @@ -1,116 +1,116 @@ pipeline {{ agent {{ docker {{ - image 'swh-jenkins/{docker-image}' + image 'swh-jenkins/{docker_image}' args '--tmpfs /tmp:exec --mount type=volume,src=shared-jenkins-cachedir,dst=/home/jenkins/.cache' }} }} environment {{ PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' CYPRESS_CACHE_FOLDER = '/tmp/cypress_cache' }} stages {{ stage('Checkout') {{ steps {{ withCredentials([ string(credentialsId: 'swh-public-ci', variable: 'PHAB_CONDUIT_TOKEN')]) {{ sh ''' if [ -n "$PHID" ]; then echo "{{ \\\"buildTargetPHID\\\": \\\"$PHID\\\", \\\"artifactKey\\\": \\\"link.jenkins\\\", \\\"artifactType\\\": \\\"uri\\\", \\\"artifactData\\\": {{ \\\"uri\\\": \\\"$BUILD_URL\\\", \\\"name\\\": \\\"Jenkins\\\", \\\"ui.external\\\": true }} }}" | arc call-conduit --conduit-uri $PHAB_CONDUIT_URL --conduit-token $PHAB_CONDUIT_TOKEN harbormaster.createartifact python3 -m pyarcanist send-message work $PHID fi ''' }} checkout([$class: 'GitSCM', branches: [[name: "${{params.REVISION}}"]], doGenerateSubmoduleConfigurations: false, extensions: [], gitTool: 'Default', submoduleCfg: [], userRemoteConfigs: [[url: 'https://forge.softwareheritage.org/source/{display-name}.git']], browser: [$class: 'Phabricator', repoUrl: 'https://forge.softwareheritage.org', repo: '{name}'] ]) }} }} stage('Setup environment') {{ steps {{ sh '''#!/bin/bash python3 -m pip install --user -e .[testing] yarn install && yarn build-test && yarn run cypress install ''' }} }} stage('Run cypress tests') {{ steps {{ sh '''#!/bin/bash set -e export PYTHONPATH=$PWD python3 swh/web/manage.py migrate --settings=swh.web.settings.tests python3 swh/web/manage.py createcachetable --settings=swh.web.settings.tests cat swh/web/tests/create_test_admin.py | python3 swh/web/manage.py shell --settings=swh.web.settings.tests python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & wait-for-it localhost:5004 yarn run cypress run yarn run mochawesome yarn run nyc-report ''' }} }} }} post {{ always {{ withCredentials([ string(credentialsId: 'swh-public-ci', variable: 'PHAB_CONDUIT_TOKEN')]) {{ withEnv(["JOBSTATUS=${{currentBuild.currentResult}}"]) {{ sh ''' if [ "$JOBSTATUS" = "SUCCESS" ]; then MSGTYPE=pass else MSGTYPE=fail fi echo "Current job status is $JOBSTATUS -> $MGSTYPE" if [ -n "$PHID" ]; then python3 -m pyarcanist send-message $MSGTYPE $PHID fi ''' }} }} publishHTML (target: [ allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true, reportDir: 'cypress/mochawesome/report', reportFiles: 'mochawesome.html', reportName: "Mochawesome Tests Report" ]) publishHTML (target: [ allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true, reportDir: 'cypress/coverage/lcov-report', reportFiles: 'index.html', reportName: "Istanbul Code Coverage" ]) }} }} }} diff --git a/jobs/templates/swh-cypress.yaml b/jobs/templates/swh-cypress.yaml index 7514af9..174ec4f 100644 --- a/jobs/templates/swh-cypress.yaml +++ b/jobs/templates/swh-cypress.yaml @@ -1,98 +1,98 @@ - job-template: name: "{name}/cypress" display-name: "master branch (cypress)" project-type: pipeline - docker-image: cypress + docker_image: cypress triggers: - timed: '@daily' sandbox: true auth-token: 'ph4br1cat0r' properties: - build-discarder: num-to-keep: 20 parameters: - git-parameter: name: REVISION type: PT_REVISION defaultValue: master description: Git revision to build. - string: name: PHID description: PHID of the Phabricator target object on which results will be reported. dsl: !include-raw: swh-cypress.groovy - job-template: name: "{name}/cypress-diff" display-name: 'Phab. Diff (cypress)' - docker-image: cypress - node: "swh-{docker-image}" + docker_image: cypress + node: "swh-{docker_image}" auth-token: ph4br1cat0r parameters: - string: name: REPO description: PHID of the Diffusion repository - string: name: PHID description: PHID of the Target object - string: name: DIFF_ID description: ID of the Diff patch to apply, if any scm: - git: url: https://forge.softwareheritage.org/source/{display-name}.git branches: - origin/master wrappers: - phabricator-differential: apply-to-master: true patch-with-force-flag: true builders: - shell: | echo "Run cypress for:" echo " REPO=$REPO" echo " PHID=$PHID" echo " DIFF_ID=$DIFF_ID" - shell: | #!/bin/bash set -e python3 -m pip install --user -e .[testing] export CYPRESS_CACHE_FOLDER=/tmp/cypress_cache/ yarn install && yarn build-test && yarn run cypress install export PYTHONPATH=$PWD python3 swh/web/manage.py migrate --settings=swh.web.settings.tests python3 swh/web/manage.py createcachetable --settings=swh.web.settings.tests cat swh/web/tests/create_test_admin.py | python3 swh/web/manage.py shell --settings=swh.web.settings.tests python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & wait-for-it localhost:5004 yarn run cypress run yarn run mochawesome yarn run nyc-report publishers: - phabricator: uberalls-enabled: false comment-on-success: true comment-with-console-link-on-failure: true process-lint: true lint-file: .phabricator-lint lint-file-size: 1000000 - html-publisher: name: 'Mochawesome Tests Report' dir: 'cypress/mochawesome/report' files: 'mochawesome.html' keep-all: true allow-missing: true link-to-last-build: false - html-publisher: name: 'Istanbul Code Coverage' dir: 'cypress/coverage/lcov-report' files: 'index.html' keep-all: true allow-missing: true link-to-last-build: false diff --git a/jobs/templates/swh-pipeline.groovy b/jobs/templates/swh-pipeline.groovy index 4c9a06a..2bffc81 100644 --- a/jobs/templates/swh-pipeline.groovy +++ b/jobs/templates/swh-pipeline.groovy @@ -1,129 +1,129 @@ pipeline {{ agent {{ docker {{ - image 'swh-jenkins/{docker-image}' + image 'swh-jenkins/{docker_image}' args '--tmpfs /tmp:exec --mount type=volume,src=shared-jenkins-cachedir,dst=/home/jenkins/.cache' }} }} environment {{ PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' }} stages {{ stage('Checkout') {{ steps {{ withCredentials([ string(credentialsId: 'swh-public-ci', variable: 'PHAB_CONDUIT_TOKEN')]) {{ sh ''' if [ -n "$PHID" ]; then echo "{{ \\\"buildTargetPHID\\\": \\\"$PHID\\\", \\\"artifactKey\\\": \\\"link.jenkins\\\", \\\"artifactType\\\": \\\"uri\\\", \\\"artifactData\\\": {{ \\\"uri\\\": \\\"$BUILD_URL\\\", \\\"name\\\": \\\"Jenkins\\\", \\\"ui.external\\\": true }} }}" | arc call-conduit --conduit-uri $PHAB_CONDUIT_URL --conduit-token $PHAB_CONDUIT_TOKEN harbormaster.createartifact python3 -m pyarcanist send-message work $PHID fi ''' }} checkout([$class: 'GitSCM', branches: [[name: "${{params.REVISION}}"]], doGenerateSubmoduleConfigurations: false, extensions: [], gitTool: 'Default', submoduleCfg: [], userRemoteConfigs: [[url: 'https://forge.softwareheritage.org/source/{display-name}.git']], browser: [$class: 'Phabricator', repoUrl: 'https://forge.softwareheritage.org', repo: '{name}'] ]) }} }} stage('Static analysis') {{ steps {{ echo 'flake8' sh '''python3 -m tox -e flake8''' echo 'mypy' sh '''if python3 -m tox -a 2>/dev/null | grep -qx mypy ; then python3 -m tox -e mypy ; fi''' echo 'radon'; sh ''' mkdir -p reports python3 -m radon raw --json swh/ > reports/raw_report.json python3 -m radon cc --json swh/ > reports/cc_report.json python3 -m radon mi --json swh/ > reports/mi_report.json python3 -m radon hal --json swh/ > reports/hal_report.json python3 -m radon cc --xml swh/ > reports/cc_report.xml ''' }} }} // static analysis stage('Unit tests') {{ options {{ timeout(time: 20, unit: 'MINUTES') }} steps {{ sh ''' python3 -m tox -e $TOX_ENVIRONMENT -- \ --cov-report=xml \ --junit-xml=test-results.xml \ -v ''' }} post {{ always {{ step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: 'coverage.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 10, onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false]) // JUnit report junit allowEmptyResults: true, testResults: 'test-results.xml' // disabled for now, requires the plugin Warning v5 (still in RC) //recordIssues enabledForFailure: true, // tools: [[pattern: '**/reports/cc_report.xml', tool: [$class: 'Ccm']]] }} }} // post }} // unit tests }} // stages post {{ always {{ // Archive a few report files archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/*,*.xml,tox*.ini', fingerprint: true withCredentials([ string(credentialsId: 'swh-public-ci', variable: 'PHAB_CONDUIT_TOKEN')]) {{ withEnv(["JOBSTATUS=${{currentBuild.currentResult}}"]) {{ sh ''' if [ "$JOBSTATUS" = "SUCCESS" ]; then MSGTYPE=pass else MSGTYPE=fail fi echo "Current job status is $JOBSTATUS -> $MGSTYPE" if [ -n "$PHID" ]; then python3 -m pyarcanist send-message $MSGTYPE $PHID fi ''' }} // withEnv }} // withCredentials }} // always }} // post }} // pipeline diff --git a/jobs/templates/swh-pipeline.yaml b/jobs/templates/swh-pipeline.yaml index 8b94516..d331d4f 100644 --- a/jobs/templates/swh-pipeline.yaml +++ b/jobs/templates/swh-pipeline.yaml @@ -1,30 +1,30 @@ - job-template: name: "{name}/tests" display-name: "master branch" project-type: pipeline - docker-image: tox + docker_image: tox default-tox-environment: py3 triggers: - timed: '@daily' sandbox: true auth-token: 'ph4br1cat0r' properties: - build-discarder: num-to-keep: 20 parameters: - git-parameter: name: REVISION type: PT_REVISION defaultValue: master description: Git revision to build. - string: name: PHID description: PHID of the Phabricator target object on which results will be reported. - string: name: TOX_ENVIRONMENT description: tox environment to use for the main tox run default: "{default-tox-environment}" dsl: !include-raw: swh-pipeline.groovy diff --git a/jobs/templates/swh-pypi.groovy b/jobs/templates/swh-pypi.groovy index 04a54c0..a22d4aa 100644 --- a/jobs/templates/swh-pypi.groovy +++ b/jobs/templates/swh-pypi.groovy @@ -1,115 +1,115 @@ def module_name = '{display-name}'.replace('-', '.') def PYPI_UPLOAD_HOST switch (params.PYPI_HOST) {{ case 'pypi.org': PYPI_UPLOAD_HOST = 'upload.pypi.org' break default: PYPI_UPLOAD_HOST = params.PYPI_HOST break }} pipeline {{ agent {{ docker {{ - image 'swh-jenkins/{docker-image}' + image 'swh-jenkins/{docker_image}' args '--tmpfs /tmp:exec --mount type=volume,src=shared-jenkins-cachedir,dst=/home/jenkins/.cache' }} }} stages {{ stage('Run tests') {{ when {{ expression {{ return !params.SKIP_TESTS }} beforeAgent true }} agent none steps {{ build( job: '/{name}/tests', parameters: [ string(name: 'REVISION', value: params.GIT_TAG), ], propagate: !params.IGNORE_TESTS, ) }} }} stage('Run cypress tests') {{ when {{ expression {{ '{display-name}' == 'swh-web' && !params.SKIP_TESTS }} beforeAgent true }} agent none steps {{ build( job: '/{name}/cypress', parameters: [ string(name: 'REVISION', value: params.GIT_TAG), ], propagate: !params.IGNORE_TESTS, ) }} }} stage('Checkout') {{ steps {{ checkout([$class: 'GitSCM', branches: [[name: params.GIT_TAG]], doGenerateSubmoduleConfigurations: false, extensions: [], gitTool: 'Default', submoduleCfg: [], userRemoteConfigs: [[url: 'https://forge.softwareheritage.org/source/{display-name}.git']] ]) }} }} stage('Build') {{ steps {{ sh ''' # Build javascript assets if [ -f yarn.lock ]; then yarn install --frozen-lockfile yarn build fi # Build java assets if [ -d java ]; then for pom in $( find java/ -name pom.xml ) ; do mvn -f $pom compile assembly:single done fi python3 setup.py sdist bdist_wheel ''' archiveArtifacts allowEmptyArchive: true, artifacts: 'dist/*', fingerprint: true }} }} stage('Publish') {{ when {{ anyOf {{ expression {{ return params.FORCE_UPLOAD }} expression {{ LASTV=sh(returnStdout: true, script:"curl -s https://${{params.PYPI_HOST}}/pypi/${{module_name}}/json | jq -r .info.version || true").trim() return 'v'+LASTV != params.GIT_TAG }} }} }} steps {{ withCredentials([ usernamePassword(credentialsId: PYPI_UPLOAD_HOST, usernameVariable: 'TWINE_USERNAME', passwordVariable: 'TWINE_PASSWORD')]) {{ sh "python3 -m twine upload --repository-url https://${{PYPI_UPLOAD_HOST}}/legacy/ dist/*" }} }} }} }} }} diff --git a/jobs/templates/swh-pypi.yaml b/jobs/templates/swh-pypi.yaml index 7f2c9a1..f231237 100644 --- a/jobs/templates/swh-pypi.yaml +++ b/jobs/templates/swh-pypi.yaml @@ -1,44 +1,44 @@ - job-template: name: "{name}/pypi-upload" display-name: "Upload to PyPI" - docker-image: tox + docker_image: tox project-type: pipeline sandbox: true properties: - build-discarder: num-to-keep: 20 parameters: - git-parameter: name: GIT_TAG description: git tag to process type: PT_TAG sortMode: DESCENDING_SMART selectedValue: TOP - choice: name: PYPI_HOST description: | PYPI registry to publish to when built revision is a tag. There must exists a usernamePassword credential object with that name. The pypi JSON API endpoint is built as https://$PYPI_HOST/project//json choices: - test.pypi.org - pypi.org default: test.pypi.org - bool: name: FORCE_UPLOAD default: false description: | Force uploading python packages on the chosen PYPI registry, even if the package already exists. - bool: name: SKIP_TESTS default: false description: | Do not run tests on the repository. - bool: name: IGNORE_TESTS default: false description: | Proceed even if the tests are failing on the repository. dsl: !include-raw: swh-pypi.groovy diff --git a/jobs/templates/swh-tox.yaml b/jobs/templates/swh-tox.yaml index e855c8b..f161d9e 100644 --- a/jobs/templates/swh-tox.yaml +++ b/jobs/templates/swh-tox.yaml @@ -1,61 +1,61 @@ - job-template: name: '{name}/tox' display-name: 'Phab. Diff' - docker-image: tox - node: "swh-{docker-image}" + docker_image: tox + node: "swh-{docker_image}" default-tox-environment: py3 auth-token: ph4br1cat0r parameters: - string: name: REPO description: PHID of the Diffusion repository - string: name: PHID description: PHID of the Target object - string: name: DIFF_ID description: ID of the Diff patch to apply, if any - string: name: TOX_ENVIRONMENT description: tox environment to use for the main tox run default: "{default-tox-environment}" scm: - git: url: https://forge.softwareheritage.org/source/{display-name}.git branches: - origin/master wrappers: - phabricator-differential: apply-to-master: true patch-with-force-flag: true builders: - shell: | echo "Run tox for:" echo " REPO=$REPO" echo " PHID=$PHID" echo " DIFF_ID=$DIFF_ID" - shell: | if (python3 -m tox -a 2>/dev/null | grep -qx flake8-phab ); then python3 -m tox -e flake8-phab else python3 -m tox -e flake8 fi - shell: | if (python3 -m tox -a 2>/dev/null | grep -qx mypy ); then python3 -m tox -e mypy fi - shell: python3 -m tox --result-json tox-results.json -e $TOX_ENVIRONMENT -- -v publishers: - phabricator: uberalls-enabled: false comment-on-success: true comment-with-console-link-on-failure: true process-lint: true lint-file: .phabricator-lint lint-file-size: 1000000 - archive: artifacts: "*.json,.coverage*,.phabricator*"