Page MenuHomeSoftware Heritage
Files F9312324

No OneTemporary
Actions

View Options

diff --git a/.gitignore b/.gitignore
index f6c78d7..11fc029 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,2 @@
.tox
-
+jenkins-token
\ No newline at end of file
diff --git a/jobs/swh-packages.yaml b/jobs/swh-packages.yaml
index 9d99d66..5d88171 100644
--- a/jobs/swh-packages.yaml
+++ b/jobs/swh-packages.yaml
@@ -1,345 +1,349 @@
- job-group:
name: "swh-jobs-{name}"
+ gitlab_url: https://gitlab-staging.swh.network
+ gitlab_connection_name: gitlab-staging
+ gitlab_project_name: "swh/devel/{repo_name}"
jobs:
- "{name}"
- "{name}/incoming-tag"
- "{name}/pypi-upload"
- "{name}/tests"
- "{name}/tests-on-diff"
+ - "{name}/gitlab-tests"
- "debian/packages/{name}"
- "debian/packages/{name}/update-for-release"
- "debian/packages/{name}/automatic-backport"
- "debian/packages/{name}/gbp-buildpackage"
- project:
name: DAUTH
display-name: swh-auth
repo_name: swh-auth
pkg: auth
python_module: swh.auth
jobs:
- "swh-jobs-{name}"
- project:
name: DCORE
display-name: swh-core
repo_name: swh-core
pkg: core
python_module: swh.core
jobs:
- "swh-jobs-{name}"
- project:
name: DCNT
display-name: swh-counters
repo_name: swh-counters
pkg: counters
python_module: swh.counters
jobs:
- "swh-jobs-{name}"
- project:
name: DDATASET
display-name: swh-dataset
repo_name: swh-dataset
pkg: dataset
python_module: swh.dataset
jobs:
- "swh-jobs-{name}"
- project:
name: DDEP
display-name: swh-deposit
repo_name: swh-deposit
pkg: deposit
python_module: swh.deposit
jobs:
- "swh-jobs-{name}"
- project:
name: DGRPH
display-name: swh-graph
repo_name: swh-graph
pkg: graph
python_module: swh.graph
jobs:
- "swh-jobs-{name}"
- project:
name: DGQL
display-name: swh-graphql
repo_name: swh-graphql
pkg: graphql
python_module: swh.graphql
jobs:
- "swh-jobs-{name}"
- project:
name: DCIDX
display-name: swh-indexer
repo_name: swh-indexer
pkg: indexer
python_module: swh.indexer
jobs:
- "swh-jobs-{name}"
- project:
name: DICP
display-name: swh-icinga-plugins
repo_name: swh-icinga-plugins
pkg: icinga-plugins
python_module: swh.icinga_plugins
jobs:
- "swh-jobs-{name}"
- project:
name: DJNL
display-name: swh-journal
repo_name: swh-journal
docker_image: kafka
pkg: journal
python_module: swh.journal
jobs:
- "swh-jobs-{name}"
- project:
name: DLS
display-name: swh-lister
repo_name: swh-lister
pkg: lister
python_module: swh.lister
jobs:
- "swh-jobs-{name}"
- project:
name: DLDBZR
display-name: swh-loader-bzr
repo_name: swh-loader-bzr
pkg: loader.bzr
python_module: swh.loader.bzr
jobs:
- "swh-jobs-{name}"
- project:
name: DLDBASE
display-name: swh-loader-core
repo_name: swh-loader-core
pkg: loader.core
python_module: swh.loader.core
jobs:
- "swh-jobs-{name}"
- project:
name: DLDCVS
display-name: swh-loader-cvs
repo_name: swh-loader-cvs
pkg: loader.cvs
python_module: swh.loader.cvs
include_bdist: false
jobs:
- "swh-jobs-{name}"
- project:
name: DLDG
display-name: swh-loader-git
repo_name: swh-loader-git
pkg: loader.git
python_module: swh.loader.git
jobs:
- "swh-jobs-{name}"
- project:
name: DLDMD
display-name: swh-loader-metadata
repo_name: swh-loader-metadata
pkg: loader.metadata
python_module: swh.loader.metadata
jobs:
- "swh-jobs-{name}"
- project:
name: DLDHG
display-name: swh-loader-mercurial
repo_name: swh-loader-mercurial
pkg: loader.mercurial
python_module: swh.loader.mercurial
jobs:
- "swh-jobs-{name}"
- project:
name: DLDSVN
display-name: swh-loader-svn
repo_name: swh-loader-svn
pkg: loader.svn
python_module: swh.loader.svn
jobs:
- "swh-jobs-{name}"
- project:
name: DMOD
display-name: swh-model
repo_name: swh-model
pkg: model
python_module: swh.model
jobs:
- "swh-jobs-{name}"
- project:
name: DMFCD
display-name: swh-clearlydefined
repo_name: swh-clearlydefined
pkg: clearlydefined
python_module: swh.clearlydefined
jobs:
- "swh-jobs-{name}"
- project:
name: DOBJS
display-name: swh-objstorage
repo_name: swh-objstorage
pkg: objstorage
deb-extra-repositories: ceph
python_module: swh.objstorage
jobs:
- "swh-jobs-{name}"
- project:
name: DOBJSRPL
display-name: swh-objstorage-replayer
repo_name: swh-objstorage-replayer
pkg: objstorage.replayer
python_module: swh.objstorage.replayer
jobs:
- "swh-jobs-{name}"
- project:
name: DOPH
display-name: swh-perfecthash
repo_name: swh-perfecthash
pkg: perfecthash
python_module: swh.perfecthash
include_bdist: false
jobs:
- "swh-jobs-{name}"
- project:
name: DPROV
display-name: swh-provenance
repo_name: swh-provenance
pkg: provenance
python_module: swh.provenance
timeout: 15
jobs:
- "swh-jobs-{name}"
- project:
name: DSEA
display-name: swh-search
repo_name: swh-search
pkg: search
deb-extra-repositories: elasticsearch
python_module: swh.search
jobs:
- "swh-jobs-{name}"
- project:
name: TSBX
display-name: swh-sandbox
repo_name: swh-sandbox
pkg: sandbox
incoming_tag_auto_pypi_host: test.pypi.org
python_module: swh.sandbox
jobs:
- "{name}"
- "{name}/incoming-tag"
- "{name}/pypi-upload"
- "{name}/tests"
- "{name}/tests-on-diff"
- project:
name: DTSCN
display-name: swh-scanner
repo_name: swh-scanner
pkg: scanner
python_module: swh.scanner
jobs:
- "swh-jobs-{name}"
- project:
name: DSCH
display-name: swh-scheduler
repo_name: swh-scheduler
pkg: scheduler
python_module: swh.scheduler
jobs:
- "swh-jobs-{name}"
- project:
name: DSCRUB
display-name: swh-scrubber
repo_name: swh-scrubber
pkg: scrubber
python_module: swh.scrubber
jobs:
- "swh-jobs-{name}"
- project:
name: DSTO
display-name: swh-storage
repo_name: swh-storage
pkg: storage
deb-extra-repositories: cassandra
python_module: swh.storage
timeout: 25
jobs:
- "swh-jobs-{name}"
- project:
name: DVAU
display-name: swh-vault
repo_name: swh-vault
pkg: vault
python_module: swh.vault
jobs:
- "swh-jobs-{name}"
- project:
name: DWAPPS
display-name: swh-web
repo_name: swh-web
pkg: web
python_module: swh.web
do_cypress: true
nb_cypress_runners: 4
timeout: 30
max_concurrent: 3
jobs:
- "swh-jobs-{name}"
- project:
name: DWCLI
display-name: swh-web-client
repo_name: swh-web-client
pkg: web-client
python_module: swh.web.client
jobs:
- "swh-jobs-{name}"
- project:
name: DFUSE
display-name: swh-fuse
repo_name: swh-fuse
pkg: fuse
python_module: swh.fuse
docker_options: --privileged --device /dev/fuse
jobs:
- "swh-jobs-{name}"
- project:
name: DTPL
display-name: swh-py-template
repo_name: swh-py-template
pkg: py-template
python_module: swh.foo
jobs:
- "swh-jobs-{name}"
diff --git a/jobs/templates/swh-pipeline-diff.yaml b/jobs/templates/swh-pipeline-diff.yaml
index 99fa975..eb40b91 100644
--- a/jobs/templates/swh-pipeline-diff.yaml
+++ b/jobs/templates/swh-pipeline-diff.yaml
@@ -1,43 +1,44 @@
- job-template:
name: "{name}/tests-on-diff"
display-name: Phab. Diff
project-type: pipeline
docker_image: tox
cypress_docker_image: cypress
sphinx_docker_image: sphinx
default-tox-environment: py3
concurrent: true
sandbox: true
auth-token: "ph4br1cat0r"
phabricator_diff: true
+ gitlab_project: false
do_cypress: false
timeout: 10
max_concurrent: 0
properties:
- build-discarder:
days-to-keep: 90
artifact-num-to-keep: 20
parameters:
- git-parameter:
name: REVISION
type: PT_REVISION
defaultValue: master
description: Git revision to build.
- string:
name: PHID
description: PHID of the Phabricator target object on which results will be reported.
- string:
name: DIFF_ID
description: ID of the Diff patch to apply, if any
- string:
name: REV_ID
description: ID of the Phabricator revision, if any
- string:
name: STAGING_URI
description: URI of the staging repository
- string:
name: TOX_ENVIRONMENT
description: tox environment to use for the main tox run
default: "{default-tox-environment}"
dsl: !include-jinja2: swh-pipeline.groovy.j2
diff --git a/jobs/templates/swh-pipeline.groovy.j2 b/jobs/templates/swh-pipeline.groovy.j2
index d0e0149..4fbb64d 100644
--- a/jobs/templates/swh-pipeline.groovy.j2
+++ b/jobs/templates/swh-pipeline.groovy.j2
@@ -1,152 +1,189 @@
pipeline {
{% filter indent(width=2) %}
{%- include 'includes/agent-docker.groovy.j2' -%}
{% endfilter %}
options {
// require "Throttle Concurrent Builds" Jenkins plugin
throttleJobProperty(
categories: [],
limitOneJobWithMatchingParams: false,
maxConcurrentPerNode: {{ max_concurrent }},
maxConcurrentTotal: {{ max_concurrent }},
paramsToUseForLimit: '',
throttleEnabled: true,
throttleOption: 'project',
)
}
+ {%- if not gitlab_project %}
environment {
PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/'
}
+ {%- endif %}
stages {
stage('Checkout') {
steps {
+ {%- if not gitlab_project %}
{% filter indent(width=8) %}
{%- include 'includes/create-phabricator-artifacts.groovy.j2' -%}
{% endfilter %}
+ {%- endif %}
checkout([
$class: 'GitSCM',
- branches: [[name: "${params.REVISION}"]],
+ {%- if gitlab_project %}
+ userRemoteConfigs: [[
+ url: '{{gitlab_url}}/{{gitlab_project_name}}.git',
+ refspec: '+refs/heads/*:refs/remotes/origin/* \
+ +refs/merge-requests/*/head:refs/remotes/origin/merge-requests/* \
+ +refs/tags/*:refs/remotes/origin/tags*',
+ ]],
+ branches: [[
+ name: params.REVISION ?: "${env.gitlabSourceBranch}"
+ ]],
+ browser: [
+ $class: 'GitLab',
+ repoUrl: '{{gitlab_url}}/{{gitlab_project_name}}'
+ ],
+ {%- else %}
+ userRemoteConfigs: [[
+ url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git',
+ refspec: '+refs/heads/master:refs/remotes/origin/master',
+ ]],
+ branches: [[
+ name: "${params.REVISION}"
+ ]],
+ browser: [
+ $class: 'Phabricator',
+ repoUrl: 'https://forge.softwareheritage.org', repo: '{{name}}'
+ ],
+ {%- endif %}
doGenerateSubmoduleConfigurations: false,
extensions: [[$class: 'CloneOption', honorRefspec: true]],
gitTool: 'Default',
submoduleCfg: [],
- userRemoteConfigs: [
- [url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git',
- refspec: '+refs/heads/master:refs/remotes/origin/master'],
- ],
- browser: [$class: 'Phabricator', repoUrl: 'https://forge.softwareheritage.org', repo: '{{name}}']
])
}
}
- {%- if phabricator_diff %}
+ {%- if not gitlab_project and phabricator_diff %}
stage('Apply phabricator diff') {
steps {
{% filter indent(width=8) %}
{%- include 'includes/script-apply-phabricator-diff.groovy.j2' -%}
{% endfilter %}
}
}
{%- endif %}
stage ('flake8') {
steps {
sh '''python3 -m tox -e flake8'''
}
}
stage ('mypy') {
steps {
sh '''python3 -m tox -e mypy'''
}
}
stage ('radon') {
steps {
sh '''
mkdir -p reports
python3 -m radon raw --json swh/ > reports/raw_report.json
python3 -m radon cc --json swh/ > reports/cc_report.json
python3 -m radon mi --json swh/ > reports/mi_report.json
python3 -m radon hal --json swh/ > reports/hal_report.json
python3 -m radon cc --xml swh/ > reports/cc_report.xml
'''
}
post {
always {
// Archive a few report files
archiveArtifacts(
allowEmptyArchive: true,
artifacts: 'reports/*',
fingerprint: true,
)
// Warnings NG
recordIssues(
enabledForFailure: true,
tools: [
ccm(pattern: '**/reports/cc_report.xml'),
],
)
}
}
}
stage('Tests') {
options {
timeout(time: {{ timeout }}, unit: 'MINUTES')
}
parallel {
{% filter indent(width=8) %}
{%- include 'includes/stage-python-tests.groovy.j2' -%}
{% endfilter %}
{%- if do_cypress %}
{% filter indent(width=8) %}
{%- include 'includes/stage-cypress-tests.groovy.j2' -%}
{% endfilter %}
{%- endif %}
stage('Sphinx documentation') {
{% filter indent(width=10) %}
{%- include 'includes/agent-docker-sphinx.groovy.j2' -%}
{% endfilter %}
steps {
sh '''
if tox -a | grep -x sphinx >/dev/null
then
tox -e sphinx
else
echo WARNING: no sphinx environment in tox.ini
fi
'''
}
} // sphinx doc
} // parallel
} // Tests stage
} // stages
post {
+ {%- if gitlab_project %}
+ failure {
+ updateGitlabCommitStatus name: 'jenkins', state: 'failed'
+ }
+ success {
+ updateGitlabCommitStatus name: 'jenkins', state: 'success'
+ }
+ aborted {
+ updateGitlabCommitStatus name: 'jenkins', state: 'canceled'
+ }
+ {%- else %}
always {
step([$class: 'PhabricatorNotifier',
commentOnSuccess: true,
commentWithConsoleLinkOnFailure: true,
commentFile: '.phabricator-comment',
commentSize: '1000000',
preserveFormatting: true,
processLint: true,
lintFile: '.phabricator-lint',
lintFileSize: '1000000',
])
} // always
+ {%- endif %}
cleanup {
cleanWs()
}
} // post
} // pipeline
diff --git a/jobs/templates/swh-pipeline.yaml b/jobs/templates/swh-pipeline.yaml
index 053dd9b..22a5741 100644
--- a/jobs/templates/swh-pipeline.yaml
+++ b/jobs/templates/swh-pipeline.yaml
@@ -1,36 +1,63 @@
-- job-template:
+- job-template: &master_tests
name: "{name}/tests"
display-name: "master branch"
project-type: pipeline
docker_image: tox
cypress_docker_image: cypress
sphinx_docker_image: sphinx
default-tox-environment: py3
concurrent: true
triggers:
- timed: "@daily"
sandbox: true
auth-token: "ph4br1cat0r"
properties:
- build-discarder:
days-to-keep: 90
artifact-num-to-keep: 20
phabricator_diff: false
+ gitlab_project: false
do_cypress: false
timeout: 10
max_concurrent: 0
parameters:
- git-parameter:
name: REVISION
type: PT_REVISION
defaultValue: master
description: Git revision to build.
- string:
name: PHID
description: PHID of the Phabricator target object on which results will be reported.
- string:
name: TOX_ENVIRONMENT
description: tox environment to use for the main tox run
default: "{default-tox-environment}"
dsl: !include-jinja2: swh-pipeline.groovy.j2
+
+- job-template:
+ name: "{name}/gitlab-tests"
+ display-name: "gitlab builds"
+ auth-token:
+ properties:
+ - gitlab:
+ connection: "{gitlab_connection_name}"
+ triggers:
+ - gitlab:
+ trigger-push: true
+ trigger-merge-request: true
+ add-ci-message: true
+ cancel-pending-builds-on-update: true
+ # secret jenkins token is generated when executing tox
+ secret-token: !include-raw: jenkins-token
+ gitlab_project: true
+ parameters:
+ - string:
+ name: REVISION
+ description: Git revision to build.
+ - string:
+ name: TOX_ENVIRONMENT
+ description: tox environment to use for the main tox run
+ default: "{default-tox-environment}"
+ <<: *master_tests
diff --git a/tox.ini b/tox.ini
index 7466e87..0616c2b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,12 +1,16 @@
[tox]
skipsdist = true
envlist = test
[testenv]
basepython = python3
skip_install = true
deps =
jenkins-job-builder
+
[testenv:test]
+allowlist_externals = bash
commands =
+ # generate secret jenkins token to trigger builds from gitlab webhooks
+ bash -c "echo -n $(openssl rand -base64 32 | sha1sum | cut -c1-40) > jobs/templates/jenkins-token"
jenkins-jobs {posargs:test} -r jobs

File Metadata

Mime Type
text/x-diff
Expires
Thu, Jul 3, 10:49 AM (1 w, 5 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3312581

Event Timeline

About · Developer information · Legal