diff --git a/jobs/swh-packages.yaml b/jobs/swh-packages.yaml index d8a2773..1f511a9 100644 --- a/jobs/swh-packages.yaml +++ b/jobs/swh-packages.yaml @@ -1,194 +1,195 @@ - job-group: name: 'swh-jobs-{name}' jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - '{name}/tests-on-diff' - "{name}/tox" - "debian/packages/{name}" - "debian/packages/{name}/update-for-release" - "debian/packages/{name}/automatic-backport" - "debian/packages/{name}/gbp-buildpackage" - project: name: DCORE display-name: swh-core repo_name: swh-core pkg: core jobs: - 'swh-jobs-{name}' - project: name: DDEP display-name: swh-deposit repo_name: swh-deposit pkg: deposit jobs: - 'swh-jobs-{name}' - project: name: DGRPH display-name: swh-graph repo_name: swh-graph pkg: graph jobs: - 'swh-jobs-{name}' - project: name: DCIDX display-name: swh-indexer repo_name: swh-indexer pkg: indexer jobs: - 'swh-jobs-{name}' - project: name: DICP display-name: swh-icinga-plugins repo_name: swh-icinga-plugins pkg: icinga-plugins jobs: - 'swh-jobs-{name}' - project: name: DJNL display-name: swh-journal repo_name: swh-journal docker_image: kafka pkg: journal jobs: - 'swh-jobs-{name}' - project: name: DLS display-name: swh-lister repo_name: swh-lister pkg: lister jobs: - 'swh-jobs-{name}' - project: name: DLDBASE display-name: swh-loader-core repo_name: swh-loader-core pkg: loader.core jobs: - 'swh-jobs-{name}' - project: name: DLDG display-name: swh-loader-git repo_name: swh-loader-git pkg: loader.git jobs: - 'swh-jobs-{name}' - project: name: DLDHG display-name: swh-loader-mercurial repo_name: swh-loader-mercurial pkg: loader.mercurial jobs: - 'swh-jobs-{name}' - project: name: DLDSVN display-name: swh-loader-svn repo_name: swh-loader-svn pkg: loader.svn jobs: - 'swh-jobs-{name}' - project: name: DMOD display-name: swh-model repo_name: swh-model pkg: model jobs: - 'swh-jobs-{name}' - project: name: DOBJS display-name: swh-objstorage repo_name: swh-objstorage pkg: objstorage deb-extra-repositories: ceph jobs: - 'swh-jobs-{name}' - project: name: DSEA display-name: swh-search repo_name: swh-search pkg: search deb-extra-repositories: elasticsearch jobs: - 'swh-jobs-{name}' - project: name: TSBX display-name: swh-sandbox repo_name: swh-sandbox pkg: sandbox incoming_tag_auto_pypi_host: test.pypi.org jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - '{name}/tests-on-diff' - "{name}/tox" - project: name: DTSCN display-name: swh-scanner repo_name: swh-scanner pkg: scanner jobs: - 'swh-jobs-{name}' - project: name: DSCH display-name: swh-scheduler repo_name: swh-scheduler pkg: scheduler jobs: - 'swh-jobs-{name}' - project: name: DSTO display-name: swh-storage repo_name: swh-storage pkg: storage deb-extra-repositories: cassandra jobs: - 'swh-jobs-{name}' - project: name: DVAU display-name: swh-vault repo_name: swh-vault pkg: vault jobs: - 'swh-jobs-{name}' - project: name: DWAPPS display-name: swh-web repo_name: swh-web pkg: web + do_cypress: true jobs: - 'swh-jobs-{name}' - '{name}/cypress' - '{name}/cypress-diff' - project: name: DWCLI display-name: swh-web-client repo_name: swh-web-client pkg: web-client jobs: - 'swh-jobs-{name}' diff --git a/jobs/templates/includes/stage-cypress-tests.groovy.j2 b/jobs/templates/includes/stage-cypress-tests.groovy.j2 new file mode 100644 index 0000000..68e3247 --- /dev/null +++ b/jobs/templates/includes/stage-cypress-tests.groovy.j2 @@ -0,0 +1,52 @@ +stage('Setup environment') { + agent { + docker { + image 'swh-jenkins/{{cypress_docker_image}}' + args '--tmpfs /tmp:exec --mount type=volume,src=shared-jenkins-cachedir,dst=/home/jenkins/.cache' + } + } + + steps { + echo 'Setup cypress environment' + + sh '''#!/bin/bash + python3 -m pip install --user -e .[testing] + yarn install && yarn build-test && yarn run cypress install + ''' + + echo 'Run cypress tests' + sh '''#!/bin/bash + set -e + export PYTHONPATH=$PWD + python3 swh/web/manage.py migrate --settings=swh.web.settings.tests + python3 swh/web/manage.py createcachetable --settings=swh.web.settings.tests + cat swh/web/tests/create_test_admin.py | python3 swh/web/manage.py shell --settings=swh.web.settings.tests + python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & + wait-for-it localhost:5004 + yarn run cypress run + yarn run mochawesome + yarn run nyc-report + ''' + } + + post { + always { + publishHTML (target: [ + allowMissing: true, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'cypress/mochawesome/report', + reportFiles: 'mochawesome.html', + reportName: "Mochawesome Tests Report" + ]) + publishHTML (target: [ + allowMissing: true, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'cypress/coverage/lcov-report', + reportFiles: 'index.html', + reportName: "Istanbul Code Coverage" + ]) + } + } +} diff --git a/jobs/templates/swh-pipeline-diff.yaml b/jobs/templates/swh-pipeline-diff.yaml index b51f82b..bfb2596 100644 --- a/jobs/templates/swh-pipeline-diff.yaml +++ b/jobs/templates/swh-pipeline-diff.yaml @@ -1,35 +1,37 @@ - job-template: name: "{name}/tests-on-diff" display-name: Phab. Diff project-type: pipeline docker_image: tox + cypress_docker_image: cypress default-tox-environment: py3 sandbox: true auth-token: 'ph4br1cat0r' phabricator_diff: true + do_cypress: false parameters: - git-parameter: name: REVISION type: PT_REVISION defaultValue: master description: Git revision to build. - string: name: PHID description: PHID of the Phabricator target object on which results will be reported. - string: name: DIFF_ID description: ID of the Diff patch to apply, if any - string: name: REV_ID description: ID of the Phabricator revision, if any - string: name: STAGING_URI description: URI of the staging repository - string: name: TOX_ENVIRONMENT description: tox environment to use for the main tox run default: "{default-tox-environment}" dsl: !include-jinja2: swh-pipeline.groovy.j2 diff --git a/jobs/templates/swh-pipeline.groovy.j2 b/jobs/templates/swh-pipeline.groovy.j2 index 9e5313c..e967f91 100644 --- a/jobs/templates/swh-pipeline.groovy.j2 +++ b/jobs/templates/swh-pipeline.groovy.j2 @@ -1,106 +1,116 @@ pipeline { agent { docker { image 'swh-jenkins/{{docker_image}}' args '--tmpfs /tmp:exec --mount type=volume,src=shared-jenkins-cachedir,dst=/home/jenkins/.cache' } } environment { PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' } stages { stage('Checkout') { steps { {% filter indent(width=8) %} {%- include 'includes/create-phabricator-artifacts.groovy.j2' -%} {% endfilter %} checkout([ $class: 'GitSCM', branches: [[name: "${params.REVISION}"]], doGenerateSubmoduleConfigurations: false, extensions: [], gitTool: 'Default', submoduleCfg: [], userRemoteConfigs: [ [url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git'], ], browser: [$class: 'Phabricator', repoUrl: 'https://forge.softwareheritage.org', repo: '{{name}}'] ]) } } {%- if phabricator_diff %} {% filter indent(width=4) %} {%- include 'includes/stage-apply-phabricator-diff.groovy.j2' -%} {% endfilter %} {%- endif %} stage('Cleanup') { steps { {% filter indent(width=8) %} {%- include 'includes/cleanup-workspace.groovy.j2' -%} {% endfilter %} } } stage('Static analysis') { steps { echo 'flake8' sh '''python3 -m tox -e flake8''' echo 'mypy' sh '''if python3 -m tox -a 2>/dev/null | grep -qx mypy ; then python3 -m tox -e mypy ; fi''' echo 'radon'; sh ''' mkdir -p reports python3 -m radon raw --json swh/ > reports/raw_report.json python3 -m radon cc --json swh/ > reports/cc_report.json python3 -m radon mi --json swh/ > reports/mi_report.json python3 -m radon hal --json swh/ > reports/hal_report.json python3 -m radon cc --xml swh/ > reports/cc_report.xml ''' } post { always { // Warnings NG recordIssues( enabledForFailure: true, tools: [ ccm(pattern: '**/reports/cc_report.xml'), ], ) } } } // static analysis - {% filter indent(width=4) %} - {%- include 'includes/stage-python-tests.groovy.j2' -%} - {% endfilter %} + stage('Tests') { + parallel { + {% filter indent(width=8) %} + {%- include 'includes/stage-python-tests.groovy.j2' -%} + {% endfilter %} + + {%- if do_cypress %} + {% filter indent(width=8) %} + {%- include 'includes/stage-cypress-tests.groovy.j2' -%} + {% endfilter %} + {%- endif %} + } + } } // stages post { always { // Archive a few report files archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/*,*.xml,tox*.ini', fingerprint: true step([$class: 'PhabricatorNotifier', commentOnSuccess: true, commentWithConsoleLinkOnFailure: true, commentFile: '.phabricator-comment', commentSize: '1000000', preserveFormatting: true, processLint: true, lintFile: '.phabricator-lint', lintFileSize: '1000000', ]) {% filter indent(width=6) %} {%- include 'includes/cleanup-workspace.groovy.j2' -%} {% endfilter %} } // always } // post } // pipeline diff --git a/jobs/templates/swh-pipeline.yaml b/jobs/templates/swh-pipeline.yaml index d0010fc..273b3ee 100644 --- a/jobs/templates/swh-pipeline.yaml +++ b/jobs/templates/swh-pipeline.yaml @@ -1,31 +1,33 @@ - job-template: name: "{name}/tests" display-name: "master branch" project-type: pipeline docker_image: tox + cypress_docker_image: cypress default-tox-environment: py3 triggers: - timed: '@daily' sandbox: true auth-token: 'ph4br1cat0r' properties: - build-discarder: num-to-keep: 20 phabricator_diff: false + do_cypress: false parameters: - git-parameter: name: REVISION type: PT_REVISION defaultValue: master description: Git revision to build. - string: name: PHID description: PHID of the Phabricator target object on which results will be reported. - string: name: TOX_ENVIRONMENT description: tox environment to use for the main tox run default: "{default-tox-environment}" dsl: !include-jinja2: swh-pipeline.groovy.j2