diff --git a/.gitignore b/.gitignore index f6c78d7..11fc029 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ .tox - +jenkins-token \ No newline at end of file diff --git a/jobs/swh-packages.yaml b/jobs/swh-packages.yaml index 9d99d66..5d88171 100644 --- a/jobs/swh-packages.yaml +++ b/jobs/swh-packages.yaml @@ -1,345 +1,349 @@ - job-group: name: "swh-jobs-{name}" + gitlab_url: https://gitlab-staging.swh.network + gitlab_connection_name: gitlab-staging + gitlab_project_name: "swh/devel/{repo_name}" jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tests-on-diff" + - "{name}/gitlab-tests" - "debian/packages/{name}" - "debian/packages/{name}/update-for-release" - "debian/packages/{name}/automatic-backport" - "debian/packages/{name}/gbp-buildpackage" - project: name: DAUTH display-name: swh-auth repo_name: swh-auth pkg: auth python_module: swh.auth jobs: - "swh-jobs-{name}" - project: name: DCORE display-name: swh-core repo_name: swh-core pkg: core python_module: swh.core jobs: - "swh-jobs-{name}" - project: name: DCNT display-name: swh-counters repo_name: swh-counters pkg: counters python_module: swh.counters jobs: - "swh-jobs-{name}" - project: name: DDATASET display-name: swh-dataset repo_name: swh-dataset pkg: dataset python_module: swh.dataset jobs: - "swh-jobs-{name}" - project: name: DDEP display-name: swh-deposit repo_name: swh-deposit pkg: deposit python_module: swh.deposit jobs: - "swh-jobs-{name}" - project: name: DGRPH display-name: swh-graph repo_name: swh-graph pkg: graph python_module: swh.graph jobs: - "swh-jobs-{name}" - project: name: DGQL display-name: swh-graphql repo_name: swh-graphql pkg: graphql python_module: swh.graphql jobs: - "swh-jobs-{name}" - project: name: DCIDX display-name: swh-indexer repo_name: swh-indexer pkg: indexer python_module: swh.indexer jobs: - "swh-jobs-{name}" - project: name: DICP display-name: swh-icinga-plugins repo_name: swh-icinga-plugins pkg: icinga-plugins python_module: swh.icinga_plugins jobs: - "swh-jobs-{name}" - project: name: DJNL display-name: swh-journal repo_name: swh-journal docker_image: kafka pkg: journal python_module: swh.journal jobs: - "swh-jobs-{name}" - project: name: DLS display-name: swh-lister repo_name: swh-lister pkg: lister python_module: swh.lister jobs: - "swh-jobs-{name}" - project: name: DLDBZR display-name: swh-loader-bzr repo_name: swh-loader-bzr pkg: loader.bzr python_module: swh.loader.bzr jobs: - "swh-jobs-{name}" - project: name: DLDBASE display-name: swh-loader-core repo_name: swh-loader-core pkg: loader.core python_module: swh.loader.core jobs: - "swh-jobs-{name}" - project: name: DLDCVS display-name: swh-loader-cvs repo_name: swh-loader-cvs pkg: loader.cvs python_module: swh.loader.cvs include_bdist: false jobs: - "swh-jobs-{name}" - project: name: DLDG display-name: swh-loader-git repo_name: swh-loader-git pkg: loader.git python_module: swh.loader.git jobs: - "swh-jobs-{name}" - project: name: DLDMD display-name: swh-loader-metadata repo_name: swh-loader-metadata pkg: loader.metadata python_module: swh.loader.metadata jobs: - "swh-jobs-{name}" - project: name: DLDHG display-name: swh-loader-mercurial repo_name: swh-loader-mercurial pkg: loader.mercurial python_module: swh.loader.mercurial jobs: - "swh-jobs-{name}" - project: name: DLDSVN display-name: swh-loader-svn repo_name: swh-loader-svn pkg: loader.svn python_module: swh.loader.svn jobs: - "swh-jobs-{name}" - project: name: DMOD display-name: swh-model repo_name: swh-model pkg: model python_module: swh.model jobs: - "swh-jobs-{name}" - project: name: DMFCD display-name: swh-clearlydefined repo_name: swh-clearlydefined pkg: clearlydefined python_module: swh.clearlydefined jobs: - "swh-jobs-{name}" - project: name: DOBJS display-name: swh-objstorage repo_name: swh-objstorage pkg: objstorage deb-extra-repositories: ceph python_module: swh.objstorage jobs: - "swh-jobs-{name}" - project: name: DOBJSRPL display-name: swh-objstorage-replayer repo_name: swh-objstorage-replayer pkg: objstorage.replayer python_module: swh.objstorage.replayer jobs: - "swh-jobs-{name}" - project: name: DOPH display-name: swh-perfecthash repo_name: swh-perfecthash pkg: perfecthash python_module: swh.perfecthash include_bdist: false jobs: - "swh-jobs-{name}" - project: name: DPROV display-name: swh-provenance repo_name: swh-provenance pkg: provenance python_module: swh.provenance timeout: 15 jobs: - "swh-jobs-{name}" - project: name: DSEA display-name: swh-search repo_name: swh-search pkg: search deb-extra-repositories: elasticsearch python_module: swh.search jobs: - "swh-jobs-{name}" - project: name: TSBX display-name: swh-sandbox repo_name: swh-sandbox pkg: sandbox incoming_tag_auto_pypi_host: test.pypi.org python_module: swh.sandbox jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tests-on-diff" - project: name: DTSCN display-name: swh-scanner repo_name: swh-scanner pkg: scanner python_module: swh.scanner jobs: - "swh-jobs-{name}" - project: name: DSCH display-name: swh-scheduler repo_name: swh-scheduler pkg: scheduler python_module: swh.scheduler jobs: - "swh-jobs-{name}" - project: name: DSCRUB display-name: swh-scrubber repo_name: swh-scrubber pkg: scrubber python_module: swh.scrubber jobs: - "swh-jobs-{name}" - project: name: DSTO display-name: swh-storage repo_name: swh-storage pkg: storage deb-extra-repositories: cassandra python_module: swh.storage timeout: 25 jobs: - "swh-jobs-{name}" - project: name: DVAU display-name: swh-vault repo_name: swh-vault pkg: vault python_module: swh.vault jobs: - "swh-jobs-{name}" - project: name: DWAPPS display-name: swh-web repo_name: swh-web pkg: web python_module: swh.web do_cypress: true nb_cypress_runners: 4 timeout: 30 max_concurrent: 3 jobs: - "swh-jobs-{name}" - project: name: DWCLI display-name: swh-web-client repo_name: swh-web-client pkg: web-client python_module: swh.web.client jobs: - "swh-jobs-{name}" - project: name: DFUSE display-name: swh-fuse repo_name: swh-fuse pkg: fuse python_module: swh.fuse docker_options: --privileged --device /dev/fuse jobs: - "swh-jobs-{name}" - project: name: DTPL display-name: swh-py-template repo_name: swh-py-template pkg: py-template python_module: swh.foo jobs: - "swh-jobs-{name}" diff --git a/jobs/templates/swh-pipeline-diff.yaml b/jobs/templates/swh-pipeline-diff.yaml index 99fa975..eb40b91 100644 --- a/jobs/templates/swh-pipeline-diff.yaml +++ b/jobs/templates/swh-pipeline-diff.yaml @@ -1,43 +1,44 @@ - job-template: name: "{name}/tests-on-diff" display-name: Phab. Diff project-type: pipeline docker_image: tox cypress_docker_image: cypress sphinx_docker_image: sphinx default-tox-environment: py3 concurrent: true sandbox: true auth-token: "ph4br1cat0r" phabricator_diff: true + gitlab_project: false do_cypress: false timeout: 10 max_concurrent: 0 properties: - build-discarder: days-to-keep: 90 artifact-num-to-keep: 20 parameters: - git-parameter: name: REVISION type: PT_REVISION defaultValue: master description: Git revision to build. - string: name: PHID description: PHID of the Phabricator target object on which results will be reported. - string: name: DIFF_ID description: ID of the Diff patch to apply, if any - string: name: REV_ID description: ID of the Phabricator revision, if any - string: name: STAGING_URI description: URI of the staging repository - string: name: TOX_ENVIRONMENT description: tox environment to use for the main tox run default: "{default-tox-environment}" dsl: !include-jinja2: swh-pipeline.groovy.j2 diff --git a/jobs/templates/swh-pipeline.groovy.j2 b/jobs/templates/swh-pipeline.groovy.j2 index d0e0149..e739b27 100644 --- a/jobs/templates/swh-pipeline.groovy.j2 +++ b/jobs/templates/swh-pipeline.groovy.j2 @@ -1,152 +1,191 @@ pipeline { {% filter indent(width=2) %} {%- include 'includes/agent-docker.groovy.j2' -%} {% endfilter %} options { // require "Throttle Concurrent Builds" Jenkins plugin throttleJobProperty( categories: [], limitOneJobWithMatchingParams: false, maxConcurrentPerNode: {{ max_concurrent }}, maxConcurrentTotal: {{ max_concurrent }}, paramsToUseForLimit: '', throttleEnabled: true, throttleOption: 'project', ) } + {%- if not gitlab_project %} environment { PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' } + {%- endif %} stages { stage('Checkout') { steps { + {%- if not gitlab_project %} {% filter indent(width=8) %} {%- include 'includes/create-phabricator-artifacts.groovy.j2' -%} {% endfilter %} + {%- else %} + updateGitlabCommitStatus name: 'jenkins', state: 'running' + {%- endif %} checkout([ $class: 'GitSCM', - branches: [[name: "${params.REVISION}"]], + {%- if gitlab_project %} + userRemoteConfigs: [[ + url: '{{gitlab_url}}/{{gitlab_project_name}}.git', + refspec: '+refs/heads/*:refs/remotes/origin/* \ + +refs/merge-requests/*/head:refs/remotes/origin/merge-requests/* \ + +refs/tags/*:refs/remotes/origin/tags*', + ]], + branches: [[ + name: params.REVISION ?: "${env.gitlabSourceBranch}" + ]], + browser: [ + $class: 'GitLab', + repoUrl: '{{gitlab_url}}/{{gitlab_project_name}}' + ], + {%- else %} + userRemoteConfigs: [[ + url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git', + refspec: '+refs/heads/master:refs/remotes/origin/master', + ]], + branches: [[ + name: "${params.REVISION}" + ]], + browser: [ + $class: 'Phabricator', + repoUrl: 'https://forge.softwareheritage.org', repo: '{{name}}' + ], + {%- endif %} doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true]], gitTool: 'Default', submoduleCfg: [], - userRemoteConfigs: [ - [url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git', - refspec: '+refs/heads/master:refs/remotes/origin/master'], - ], - browser: [$class: 'Phabricator', repoUrl: 'https://forge.softwareheritage.org', repo: '{{name}}'] ]) } } - {%- if phabricator_diff %} + {%- if not gitlab_project and phabricator_diff %} stage('Apply phabricator diff') { steps { {% filter indent(width=8) %} {%- include 'includes/script-apply-phabricator-diff.groovy.j2' -%} {% endfilter %} } } {%- endif %} stage ('flake8') { steps { sh '''python3 -m tox -e flake8''' } } stage ('mypy') { steps { sh '''python3 -m tox -e mypy''' } } stage ('radon') { steps { sh ''' mkdir -p reports python3 -m radon raw --json swh/ > reports/raw_report.json python3 -m radon cc --json swh/ > reports/cc_report.json python3 -m radon mi --json swh/ > reports/mi_report.json python3 -m radon hal --json swh/ > reports/hal_report.json python3 -m radon cc --xml swh/ > reports/cc_report.xml ''' } post { always { // Archive a few report files archiveArtifacts( allowEmptyArchive: true, artifacts: 'reports/*', fingerprint: true, ) // Warnings NG recordIssues( enabledForFailure: true, tools: [ ccm(pattern: '**/reports/cc_report.xml'), ], ) } } } stage('Tests') { options { timeout(time: {{ timeout }}, unit: 'MINUTES') } parallel { {% filter indent(width=8) %} {%- include 'includes/stage-python-tests.groovy.j2' -%} {% endfilter %} {%- if do_cypress %} {% filter indent(width=8) %} {%- include 'includes/stage-cypress-tests.groovy.j2' -%} {% endfilter %} {%- endif %} stage('Sphinx documentation') { {% filter indent(width=10) %} {%- include 'includes/agent-docker-sphinx.groovy.j2' -%} {% endfilter %} steps { sh ''' if tox -a | grep -x sphinx >/dev/null then tox -e sphinx else echo WARNING: no sphinx environment in tox.ini fi ''' } } // sphinx doc } // parallel } // Tests stage } // stages post { + {%- if gitlab_project %} + failure { + updateGitlabCommitStatus name: 'jenkins', state: 'failed' + } + success { + updateGitlabCommitStatus name: 'jenkins', state: 'success' + } + aborted { + updateGitlabCommitStatus name: 'jenkins', state: 'canceled' + } + {%- else %} always { step([$class: 'PhabricatorNotifier', commentOnSuccess: true, commentWithConsoleLinkOnFailure: true, commentFile: '.phabricator-comment', commentSize: '1000000', preserveFormatting: true, processLint: true, lintFile: '.phabricator-lint', lintFileSize: '1000000', ]) } // always + {%- endif %} cleanup { cleanWs() } } // post } // pipeline diff --git a/jobs/templates/swh-pipeline.yaml b/jobs/templates/swh-pipeline.yaml index 053dd9b..f1ecdbe 100644 --- a/jobs/templates/swh-pipeline.yaml +++ b/jobs/templates/swh-pipeline.yaml @@ -1,36 +1,63 @@ -- job-template: +- job-template: &master_tests name: "{name}/tests" display-name: "master branch" project-type: pipeline docker_image: tox cypress_docker_image: cypress sphinx_docker_image: sphinx default-tox-environment: py3 concurrent: true triggers: - timed: "@daily" sandbox: true auth-token: "ph4br1cat0r" properties: - build-discarder: days-to-keep: 90 artifact-num-to-keep: 20 phabricator_diff: false + gitlab_project: false do_cypress: false timeout: 10 max_concurrent: 0 parameters: - git-parameter: name: REVISION type: PT_REVISION defaultValue: master description: Git revision to build. - string: name: PHID description: PHID of the Phabricator target object on which results will be reported. - string: name: TOX_ENVIRONMENT description: tox environment to use for the main tox run default: "{default-tox-environment}" dsl: !include-jinja2: swh-pipeline.groovy.j2 + +- job-template: + name: "{name}/gitlab-tests" + display-name: "GitLab builds" + auth-token: + properties: + - gitlab: + connection: "{gitlab_connection_name}" + triggers: + - gitlab: + trigger-push: true + trigger-merge-request: true + add-ci-message: true + cancel-pending-builds-on-update: true + # secret jenkins token is generated when executing tox + secret-token: !include-raw: jenkins-token + gitlab_project: true + parameters: + - string: + name: REVISION + description: Git revision to build. + - string: + name: TOX_ENVIRONMENT + description: tox environment to use for the main tox run + default: "{default-tox-environment}" + <<: *master_tests diff --git a/tox.ini b/tox.ini index 7466e87..0616c2b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,16 @@ [tox] skipsdist = true envlist = test [testenv] basepython = python3 skip_install = true deps = jenkins-job-builder + [testenv:test] +allowlist_externals = bash commands = + # generate secret jenkins token to trigger builds from gitlab webhooks + bash -c "echo -n $(openssl rand -base64 32 | sha1sum | cut -c1-40) > jobs/templates/jenkins-token" jenkins-jobs {posargs:test} -r jobs