diff --git a/jobs/swh-packages.yaml b/jobs/swh-packages.yaml index 05dd130..e158f47 100644 --- a/jobs/swh-packages.yaml +++ b/jobs/swh-packages.yaml @@ -1,166 +1,169 @@ - job-group: name: 'swh-jobs-{name}' jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tox" - "debian/packages/{name}" - "debian/packages/{name}/update-for-release" - "debian/packages/{name}/automatic-backport" - "debian/packages/{name}/gbp-buildpackage" - project: name: DCORE display-name: swh-core pkg: core jobs: - 'swh-jobs-{name}' - project: name: DDEP display-name: swh-deposit pkg: deposit jobs: - 'swh-jobs-{name}' - project: name: DCIDX display-name: swh-indexer pkg: indexer jobs: - 'swh-jobs-{name}' - project: name: DJNL display-name: swh-journal docker-image: swh-kafka pkg: journal jobs: - 'swh-jobs-{name}' - project: name: DLS display-name: swh-lister pkg: lister jobs: - 'swh-jobs-{name}' - project: name: DLDBASE display-name: swh-loader-core pkg: loader.core jobs: - 'swh-jobs-{name}' - project: name: DLDDEB display-name: swh-loader-debian pkg: loader.debian jobs: - 'swh-jobs-{name}' - project: name: DLDDIR display-name: swh-loader-dir pkg: loader.dir jobs: - 'swh-jobs-{name}' - project: name: DLDG display-name: swh-loader-git pkg: loader.git jobs: - 'swh-jobs-{name}' - project: name: DLDNPM display-name: swh-loader-npm pkg: loader.npm jobs: - 'swh-jobs-{name}' - project: name: DLDHG display-name: swh-loader-mercurial pkg: loader.mercurial jobs: - 'swh-jobs-{name}' - project: name: DLDPY display-name: swh-loader-pypi pkg: loader.pypi jobs: - 'swh-jobs-{name}' - project: name: DLDSVN display-name: swh-loader-svn pkg: loader.svn jobs: - 'swh-jobs-{name}' - project: name: DLDTAR display-name: swh-loader-tar pkg: loader.tar jobs: - 'swh-jobs-{name}' - project: name: DMOD display-name: swh-model pkg: model jobs: - 'swh-jobs-{name}' - project: name: DOBJS display-name: swh-objstorage pkg: objstorage deb-extra-repositories: ceph jobs: - 'swh-jobs-{name}' - project: name: TSBX display-name: swh-sandbox pkg: sandbox incoming-tag-auto-pypi-host: test.pypi.org jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tox" - project: name: DSCH display-name: swh-scheduler pkg: scheduler jobs: - 'swh-jobs-{name}' - project: name: DSTO display-name: swh-storage pkg: storage jobs: - 'swh-jobs-{name}' - project: name: DVAU display-name: swh-vault pkg: vault jobs: - 'swh-jobs-{name}' - project: name: DWAPPS display-name: swh-web pkg: web jobs: - 'swh-jobs-{name}' + - '{name}/cypress' + - '{name}/cypress/diff' + diff --git a/jobs/templates/swh-cypress.groovy b/jobs/templates/swh-cypress.groovy new file mode 100644 index 0000000..dff4378 --- /dev/null +++ b/jobs/templates/swh-cypress.groovy @@ -0,0 +1,85 @@ +pipeline {{ + + agent {{ + agent {{ label '{docker-image}' }} + }} + + environment {{ + PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' + }} + + stages {{ + + stage('Checkout') {{ + steps {{ + withCredentials([ + string(credentialsId: 'swh-public-ci', + variable: 'PHAB_CONDUIT_TOKEN')]) {{ + sh ''' + if [ -n "$PHID" ]; then + python3 -m pyarcanist send-message work $PHID + fi + ''' + }} + checkout([$class: 'GitSCM', + branches: [[name: "${{params.REVISION}}"]], + doGenerateSubmoduleConfigurations: false, + extensions: [], + gitTool: 'Default', + submoduleCfg: [], + userRemoteConfigs: [[url: 'https://forge.softwareheritage.org/source/{display-name}.git']] + ]) + }} + }} + + stage('Setup environment') {{ + steps {{ + sh '''#!/bin/bash + python3 -m venv /swh-web-env + source ~/swh-web-env/bin/activate + pip3 install wheel + pip3 install -e .[testing] + yarn install && yarn build-dev && yarn run cypress install + ''' + }} + }} + + stage('Run cypress tests') {{ + steps {{ + sh '''#!/bin/bash + source ~/swh-web-env/bin/activate + export PYTHONPATH=$PWD + python3 swh/web/manage.py migrate + python3 swh/web/manage.py createcachetable + python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & + wait-for-it localhost:5004 + yarn run cypress run + ''' + }} + }} + + }} + + post {{ + always {{ + withCredentials([ + string(credentialsId: 'swh-public-ci', + variable: 'PHAB_CONDUIT_TOKEN')]) {{ + withEnv(["JOBSTATUS=${{currentBuild.currentResult}}"]) {{ + sh ''' + if [ "$JOBSTATUS" = "SUCCESS" ]; then + MSGTYPE=pass + else + MSGTYPE=fail + fi + echo "Current job status is $JOBSTATUS -> $MGSTYPE" + if [ -n "$PHID" ]; then + python3 -m pyarcanist send-message $MSGTYPE $PHID + fi + ''' + }} + }} + }} + }} + +}} \ No newline at end of file diff --git a/jobs/templates/swh-cypress.yaml b/jobs/templates/swh-cypress.yaml new file mode 100644 index 0000000..801cf77 --- /dev/null +++ b/jobs/templates/swh-cypress.yaml @@ -0,0 +1,80 @@ +- job-template: + name: "{name}/cypress" + display-name: "master branch (cypress)" + project-type: pipeline + docker-image: swh-cypress + triggers: + - timed: '@daily' + sandbox: true + auth-token: 'ph4br1cat0r' + properties: + - build-discarder: + num-to-keep: 20 + parameters: + - git-parameter: + name: REVISION + type: PT_REVISION + defaultValue: master + description: Git revision to build. + - string: + name: PHID + description: PHID of the Phabricator target object on which results will be reported. + + dsl: + !include-raw: swh-cypress.groovy + + +- job-template: + name: "{name}/cypress/diff" + display-name: 'Phab. Diff (cypress)' + docker-image: swh-cypress + node: "{docker-image}" + auth-token: ph4br1cat0r + parameters: + - string: + name: REPO + description: PHID of the Diffusion repository + - string: + name: PHID + description: PHID of the Target object + - string: + name: DIFF_ID + description: ID of the Diff patch to apply, if any + scm: + - git: + url: https://forge.softwareheritage.org/source/{display-name}.git + + wrappers: + - phabricator-differential: + apply-to-master: true + patch-with-force-flag: true + + + builders: + - shell: | + echo "Run cypress for:" + echo " REPO=$REPO" + echo " PHID=$PHID" + echo " DIFF_ID=$DIFF_ID" + - shell: | + #!/bin/bash + python3 -m venv ~/swh-web-env + source ~/swh-web-env/bin/activate + pip3 install wheel + pip3 install -e .[testing] + yarn install && yarn build-dev && yarn run cypress install + export PYTHONPATH=$PWD + python3 swh/web/manage.py migrate + python3 swh/web/manage.py createcachetable + python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & + wait-for-it localhost:5004 + yarn run cypress run + + publishers: + - phabricator: + uberalls-enabled: false + comment-on-success: true + comment-with-console-link-on-failure: true + process-lint: true + lint-file: .phabricator-lint + lint-file-size: 1000000 diff --git a/jobs/templates/swh-pypi.groovy b/jobs/templates/swh-pypi.groovy index afd35a9..091696e 100644 --- a/jobs/templates/swh-pypi.groovy +++ b/jobs/templates/swh-pypi.groovy @@ -1,84 +1,101 @@ def module_name = '{display-name}'.replace('-', '.') def PYPI_UPLOAD_HOST switch (params.PYPI_HOST) {{ case 'pypi.org': PYPI_UPLOAD_HOST = 'upload.pypi.org' break default: PYPI_UPLOAD_HOST = params.PYPI_HOST break }} pipeline {{ agent {{ label 'swh-tox' }} stages {{ stage('Run tests') {{ when {{ expression {{ return !params.SKIP_TESTS }} beforeAgent true }} agent none steps {{ build( job: '/{name}/tests', parameters: [ string(name: 'REVISION', value: params.GIT_TAG), ], propagate: !params.IGNORE_TESTS, ) }} }} + stage('Run cypress tests') {{ + when {{ + expression {{ '{display-name}' == 'swh-web' && !params.SKIP_TESTS }} + beforeAgent true + }} + agent none + steps {{ + build( + job: '/{name}/cypress', + parameters: [ + string(name: 'REVISION', value: params.GIT_TAG), + ], + propagate: !params.IGNORE_TESTS, + ) + }} + }} + stage('Checkout') {{ steps {{ checkout([$class: 'GitSCM', branches: [[name: params.GIT_TAG]], doGenerateSubmoduleConfigurations: false, extensions: [], gitTool: 'Default', submoduleCfg: [], userRemoteConfigs: [[url: 'https://forge.softwareheritage.org/source/{display-name}.git']] ]) }} }} stage('Build') {{ steps {{ sh ''' if [ -f yarn.lock ]; then yarn install --frozen-lockfile yarn build fi python3 setup.py sdist bdist_wheel ''' archiveArtifacts allowEmptyArchive: true, artifacts: 'dist/*', fingerprint: true }} }} stage('Publish') {{ when {{ anyOf {{ expression {{ return params.FORCE_UPLOAD }} expression {{ LASTV=sh(returnStdout: true, script:"curl -s https://${{params.PYPI_HOST}}/pypi/${{module_name}}/json | jq -r .info.version || true").trim() return 'v'+LASTV != params.GIT_TAG }} }} }} steps {{ withCredentials([ usernamePassword(credentialsId: PYPI_UPLOAD_HOST, usernameVariable: 'TWINE_USERNAME', passwordVariable: 'TWINE_PASSWORD')]) {{ sh "python3 -m twine upload --repository-url https://${{PYPI_UPLOAD_HOST}}/legacy/ dist/*" }} }} }} }} }}