No OneTemporary
Actions

Size

22 KB

Subscribers

None

View Options

	diff --git a/jobs/defaults.yaml b/jobs/defaults.yaml
	new file mode 100644
	index 0000000..d30fdca
	--- /dev/null
	+++ b/jobs/defaults.yaml
	@@ -0,0 +1,4 @@
	+- defaults:
	+ name: global
	+ gitlab_url: https://gitlab-staging.swh.network
	+ gitlab_connection_name: gitlab-staging
	diff --git a/jobs/swh-docs.yaml b/jobs/swh-docs.yaml
	index 5ff6e02..13168f6 100644
	--- a/jobs/swh-docs.yaml
	+++ b/jobs/swh-docs.yaml
	@@ -1,156 +1,177 @@
	- project:
	name: DDOC
	display-name: swh-docs
	jobs:
	- "{name}"
	- "{name}/publish"
	- "{name}/dev"
	- "{name}/build-on-diff"
	+ - "{name}/gitlab-builds"

	- job-template:
	name: "{name}/publish"
	description: Build the documentation and publish it
	node: swh-sphinx
	auth-token: ph4br1cat0r
	properties:
	- build-discarder:
	days-to-keep: 90

	triggers:
	- timed: "@midnight"

	scm:
	- git:
	url: https://forge.softwareheritage.org/source/swh-environment.git

	builders:
	- shell: \|
	#!/bin/bash
	set -ex

	crudini --del .mrconfig snippets
	crudini --del .mrconfig swh-py-template

	# Don't fetch potentially large debian branches
	sed -i '/swh-docs/!s/git clone/git clone --single-branch/g' .mrconfig

	mr -j 4 -t update
	mr -j 4 -t run sh -c 'git checkout --detach `git describe --abbrev=0 --tags`; git clean -dfx'
	cd swh-docs
	git checkout master
	git clean -dfx
	SPHINXOPTCOLOR='--no-color' tox -e sphinx

	publishers:
	- ssh:
	target: "devel"
	site: "pergamon"
	clean-remote: true
	source: "swh-docs/docs/_build/html/**"
	remove-prefix: "swh-docs/docs/_build/html"
	fail-on-error: true
	verbose: true
	- ssh:
	target: "user"
	site: "pergamon"
	clean-remote: true
	source: "swh-docs/user/_build/html/**"
	remove-prefix: "swh-docs/user/_build/html"
	fail-on-error: true
	verbose: true
	- ssh:
	target: "sysadm"
	site: "pergamon"
	clean-remote: true
	source: "swh-docs/sysadm/_build/html/**"
	remove-prefix: "swh-docs/sysadm/_build/html"
	fail-on-error: true
	verbose: true

	- job-template:
	name: "{name}/dev"
	description: Build the documentation from git repos
	node: swh-sphinx
	auth-token: ph4br1cat0r
	properties:
	- build-discarder:
	days-to-keep: 90

	triggers:
	- timed: "H 9-22/2 * * *"

	scm:
	- git:
	url: https://forge.softwareheritage.org/source/swh-environment.git

	builders:
	- shell: \|
	#!/bin/bash
	set -ex

	crudini --del .mrconfig snippets
	crudini --del .mrconfig swh-py-template

	# Don't fetch potentially large debian branches
	sed -i '/swh-docs/!s/git clone/git clone --single-branch/g' .mrconfig

	mr -j 4 -t update
	mr -t run sh -c 'echo -n "`basename $PWD` " && git describe' \| grep -v 'mr run' > revisions.txt
	cd swh-docs
	git clean -dfx
	SPHINXOPTS='-W -q --keep-going -w errors.log' SPHINXOPTCOLOR='--no-color' tox -e sphinx-dev

	publishers:
	- archive:
	artifacts: "revisions.txt,swh-docs/docs/errors.log,swh-docs/.tox/log/*"
	- html-publisher:
	name: "SWH Documentation (HEAD)"
	dir: "swh-docs/docs/_build/html"
	files: "index.html"
	keep-all: false
	includes: "*/"
	- html-publisher:
	name: "SWH User Documentation (HEAD)"
	dir: "swh-docs/user/_build/html"
	files: "index.html"
	keep-all: false
	includes: "*/"
	- html-publisher:
	name: "SWH sysadmin Documentation (HEAD)"
	dir: "swh-docs/sysadm/_build/html"
	files: "index.html"
	keep-all: false
	includes: "*/"

	-- job-template:
	+- job-template: &doc_build_on_diff
	name: "{name}/build-on-diff"
	display_name: Phab. diff
	project-type: pipeline
	docker_image: sphinx
	concurrent: true
	sandbox: true
	+ gitlab_project: false
	auth-token: "ph4br1cat0r"
	properties:
	- build-discarder:
	artifact-num-to-keep: 20
	days-to-keep: 90
	parameters:
	- git-parameter:
	name: REVISION
	type: PT_REVISION
	defaultValue: master
	description: Git revision to build.
	- string:
	name: PHID
	description: PHID of the Phabricator target object on which results will be reported.
	- string:
	name: DIFF_ID
	description: ID of the Diff patch to apply, if any
	- string:
	name: REV_ID
	description: ID of the Phabricator revision, if any
	- string:
	name: STAGING_URI
	description: URI of the staging repository

	dsl: !include-jinja2: templates/swh-docs-pipeline-diff.groovy.j2
	+
	+- job-template:
	+ name: "{name}/gitlab-builds"
	+ display_name: GitLab builds
	+ gitlab_project: true
	+ auth-token:
	+ parameters:
	+ properties:
	+ - gitlab:
	+ connection: "{gitlab_connection_name}"
	+ triggers:
	+ - gitlab:
	+ trigger-push: true
	+ trigger-merge-request: true
	+ add-ci-message: true
	+ cancel-pending-builds-on-update: true
	+ # secret jenkins token is generated when executing tox
	+ secret-token: !include-raw: jobs/templates/jenkins-token
	+ <<: *doc_build_on_diff
	diff --git a/jobs/swh-packages.yaml b/jobs/swh-packages.yaml
	index 037a7df..5ee4775 100644
	--- a/jobs/swh-packages.yaml
	+++ b/jobs/swh-packages.yaml
	@@ -1,351 +1,349 @@
	- job-group:
	name: "swh-jobs-{name}"
	- gitlab_url: https://gitlab-staging.swh.network
	- gitlab_connection_name: gitlab-staging
	gitlab_project_name: "swh/devel/{repo_name}"
	jobs:
	- "{name}"
	- "{name}/incoming-tag"
	- "{name}/pypi-upload"
	- "{name}/tests"
	- "{name}/tests-on-diff"
	- "{name}/gitlab-tests"
	- "{name}/gitlab-incoming-tag"
	- "{name}/gitlab-pypi-upload"
	- "debian/packages/{name}"
	- "debian/packages/{name}/update-for-release"
	- "debian/packages/{name}/automatic-backport"
	- "debian/packages/{name}/gbp-buildpackage"

	- project:
	name: DAUTH
	display-name: swh-auth
	repo_name: swh-auth
	pkg: auth
	python_module: swh.auth
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DCORE
	display-name: swh-core
	repo_name: swh-core
	pkg: core
	python_module: swh.core
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DCNT
	display-name: swh-counters
	repo_name: swh-counters
	pkg: counters
	python_module: swh.counters
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DDATASET
	display-name: swh-dataset
	repo_name: swh-dataset
	pkg: dataset
	python_module: swh.dataset
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DDEP
	display-name: swh-deposit
	repo_name: swh-deposit
	pkg: deposit
	python_module: swh.deposit
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DGRPH
	display-name: swh-graph
	repo_name: swh-graph
	pkg: graph
	python_module: swh.graph
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DGQL
	display-name: swh-graphql
	repo_name: swh-graphql
	pkg: graphql
	python_module: swh.graphql
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DCIDX
	display-name: swh-indexer
	repo_name: swh-indexer
	pkg: indexer
	python_module: swh.indexer
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DICP
	display-name: swh-icinga-plugins
	repo_name: swh-icinga-plugins
	pkg: icinga-plugins
	python_module: swh.icinga_plugins
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DJNL
	display-name: swh-journal
	repo_name: swh-journal
	docker_image: kafka
	pkg: journal
	python_module: swh.journal
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DLS
	display-name: swh-lister
	repo_name: swh-lister
	pkg: lister
	python_module: swh.lister
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DLDBZR
	display-name: swh-loader-bzr
	repo_name: swh-loader-bzr
	pkg: loader.bzr
	python_module: swh.loader.bzr
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DLDBASE
	display-name: swh-loader-core
	repo_name: swh-loader-core
	pkg: loader.core
	python_module: swh.loader.core
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DLDCVS
	display-name: swh-loader-cvs
	repo_name: swh-loader-cvs
	pkg: loader.cvs
	python_module: swh.loader.cvs
	include_bdist: false
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DLDG
	display-name: swh-loader-git
	repo_name: swh-loader-git
	pkg: loader.git
	python_module: swh.loader.git
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DLDMD
	display-name: swh-loader-metadata
	repo_name: swh-loader-metadata
	pkg: loader.metadata
	python_module: swh.loader.metadata
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DLDHG
	display-name: swh-loader-mercurial
	repo_name: swh-loader-mercurial
	pkg: loader.mercurial
	python_module: swh.loader.mercurial
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DLDSVN
	display-name: swh-loader-svn
	repo_name: swh-loader-svn
	pkg: loader.svn
	python_module: swh.loader.svn
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DMOD
	display-name: swh-model
	repo_name: swh-model
	pkg: model
	python_module: swh.model
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DMFCD
	display-name: swh-clearlydefined
	repo_name: swh-clearlydefined
	pkg: clearlydefined
	python_module: swh.clearlydefined
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DOBJS
	display-name: swh-objstorage
	repo_name: swh-objstorage
	pkg: objstorage
	deb-extra-repositories: ceph
	python_module: swh.objstorage
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DOBJSRPL
	display-name: swh-objstorage-replayer
	repo_name: swh-objstorage-replayer
	pkg: objstorage.replayer
	python_module: swh.objstorage.replayer
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DOPH
	display-name: swh-perfecthash
	repo_name: swh-perfecthash
	pkg: perfecthash
	python_module: swh.perfecthash
	include_bdist: false
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DPROV
	display-name: swh-provenance
	repo_name: swh-provenance
	pkg: provenance
	python_module: swh.provenance
	timeout: 15
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DSEA
	display-name: swh-search
	repo_name: swh-search
	pkg: search
	deb-extra-repositories: elasticsearch
	python_module: swh.search
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: TSBX
	display-name: swh-sandbox
	repo_name: swh-sandbox
	pkg: sandbox
	incoming_tag_auto_pypi_host: test.pypi.org
	python_module: swh.sandbox
	jobs:
	- "{name}"
	- "{name}/incoming-tag"
	- "{name}/pypi-upload"
	- "{name}/tests"
	- "{name}/tests-on-diff"

	- project:
	name: DTSCN
	display-name: swh-scanner
	repo_name: swh-scanner
	pkg: scanner
	python_module: swh.scanner
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DSCH
	display-name: swh-scheduler
	repo_name: swh-scheduler
	pkg: scheduler
	python_module: swh.scheduler
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DSCRUB
	display-name: swh-scrubber
	repo_name: swh-scrubber
	pkg: scrubber
	python_module: swh.scrubber
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DSTO
	display-name: swh-storage
	repo_name: swh-storage
	pkg: storage
	deb-extra-repositories: cassandra
	python_module: swh.storage
	timeout: 25
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DVAU
	display-name: swh-vault
	repo_name: swh-vault
	pkg: vault
	python_module: swh.vault
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DWAPPS
	display-name: swh-web
	repo_name: swh-web
	pkg: web
	python_module: swh.web
	do_cypress: true
	nb_cypress_runners: 4
	timeout: 30
	max_concurrent: 3
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DWCLI
	display-name: swh-web-client
	repo_name: swh-web-client
	pkg: web-client
	python_module: swh.web.client
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DFUSE
	display-name: swh-fuse
	repo_name: swh-fuse
	pkg: fuse
	python_module: swh.fuse
	docker_options: --privileged --device /dev/fuse
	jobs:
	- "swh-jobs-{name}"

	- project:
	name: DTPL
	display-name: swh-py-template
	repo_name: swh-py-template
	pkg: py-template
	python_module: swh.foo
	jobs:
	- "swh-jobs-{name}"
	diff --git a/jobs/templates/swh-docs-pipeline-diff.groovy.j2 b/jobs/templates/swh-docs-pipeline-diff.groovy.j2
	index b56de42..7dc72b1 100644
	--- a/jobs/templates/swh-docs-pipeline-diff.groovy.j2
	+++ b/jobs/templates/swh-docs-pipeline-diff.groovy.j2
	@@ -1,116 +1,149 @@
	pipeline {
	{% filter indent(width=2) %}
	{%- include 'templates/includes/agent-docker.groovy.j2' -%}
	{% endfilter %}

	+ {%- if not gitlab_project %}
	environment {
	PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/'
	}
	+ {%- endif %}

	stages {
	stage('Checkout swh environment') {
	steps {
	+ {%- if not gitlab_project %}
	{% filter indent(width=8) %}
	{%- include 'templates/includes/create-phabricator-artifacts.groovy.j2' -%}
	{% endfilter %}
	+ {%- else %}
	+ updateGitlabCommitStatus name: 'jenkins', state: 'running'
	+ {%- endif %}
	checkout([
	$class: 'GitSCM',
	doGenerateSubmoduleConfigurations: false,
	extensions: [[$class: 'CloneOption', depth: 1, shallow: true]],
	gitTool: 'Default',
	submoduleCfg: [],
	- userRemoteConfigs: [
	- [url: 'https://forge.softwareheritage.org/source/swh-environment.git'],
	- ],
	+ userRemoteConfigs: [[
	+ {%- if not gitlab_project %}
	+ url: 'https://forge.softwareheritage.org/source/swh-environment.git'
	+ {%- else %}
	+ url: '{{gitlab_url}}/swh/devel/swh-environment.git'
	+ {%- endif %}
	+ ]],
	])
	}
	}

	stage('Checkout swh modules') {
	steps {
	script {
	sh'''#!/bin/bash
	crudini --del .mrconfig snippets
	crudini --del .mrconfig swh-py-template
	# only fetch the HEAD branch instead of all (potentially large) branches
	sed -i '/swh-docs/!s/git clone/git clone --single-branch/g' .mrconfig
	mr -j 4 -t update
	'''
	}
	}
	}

	+ {%- if not gitlab_project %}
	stage('Apply phabricator diff') {
	steps {
	dir('swh-docs') {
	{% filter indent(width=10) %}
	{%- include 'templates/includes/script-apply-phabricator-diff.groovy.j2' -%}
	{% endfilter %}
	}
	}
	}
	+ {%- endif %}

	stage('Build Software Heritage documentation') {
	steps {
	dir('swh-docs') {
	script {
	+ {%- if gitlab_project %}
	+ if ("${env.gitlabMergeRequestIid}" != "") {
	+ sh "git fetch origin merge-requests/${env.gitlabMergeRequestIid}/head:merge_request && \
	+ git checkout merge_request"
	+ } else {
	+ sh "git checkout ${env.gitlabSourceBranch}"
	+ }
	+ {%- endif %}
	sh '''#!/bin/bash
	SPHINXOPTS='-W -q --keep-going -w errors.log' SPHINXOPTCOLOR='--no-color' tox -e sphinx-dev
	'''
	}
	}
	}
	}
	}

	post {
	+ {%- if gitlab_project %}
	+ failure {
	+ updateGitlabCommitStatus name: 'jenkins', state: 'failed'
	+ }
	+ success {
	+ updateGitlabCommitStatus name: 'jenkins', state: 'success'
	+ }
	+ aborted {
	+ updateGitlabCommitStatus name: 'jenkins', state: 'canceled'
	+ }
	+ {%- endif %}
	always {
	+ {%- if not gitlab_project %}
	step([$class: 'PhabricatorNotifier',
	commentOnSuccess: true,
	commentWithConsoleLinkOnFailure: true,
	commentFile: 'swh-docs/.phabricator-comment',
	commentSize: '1000000',
	preserveFormatting: true,
	processLint: true,
	lintFile: '.phabricator-lint',
	lintFileSize: '1000000',
	])
	+ {%- endif %}

	archiveArtifacts(
	allowEmptyArchive: true,
	artifacts: 'swh-docs/docs/errors.log,swh-docs/.tox/log/*',
	fingerprint: true,
	)

	publishHTML (target: [
	allowMissing: true,
	alwaysLinkToLastBuild: false,
	keepAll: true,
	reportDir: 'swh-docs/docs/_build/html',
	reportFiles: 'index.html',
	reportName: 'SWH Documentation'
	])

	publishHTML (target: [
	allowMissing: true,
	alwaysLinkToLastBuild: false,
	keepAll: true,
	reportDir: 'swh-docs/user/_build/html',
	reportFiles: 'index.html',
	reportName: 'SWH User Documentation'
	])

	publishHTML (target: [
	allowMissing: true,
	alwaysLinkToLastBuild: false,
	keepAll: true,
	reportDir: 'swh-docs/sysadm/_build/html',
	reportFiles: 'index.html',
	reportName: 'SWH sysadmin Documentation'
	])
	}
	cleanup {
	cleanWs()
	}
	}
	}
	diff --git a/jobs/tools/setup-gitlab-webhooks.groovy.j2 b/jobs/tools/setup-gitlab-webhooks.groovy.j2
	index 16ddb89..b8baec1 100644
	--- a/jobs/tools/setup-gitlab-webhooks.groovy.j2
	+++ b/jobs/tools/setup-gitlab-webhooks.groovy.j2
	@@ -1,86 +1,88 @@
	pipeline {
	agent any

	environment {
	GITLAB_TOKEN = credentials("jenkins-gitlab-token")
	}

	stages {

	stage('Checkout Repository') {
	steps {
	checkout([
	$class: 'GitSCM',
	branches: [[name: 'gitlab-integration']],
	userRemoteConfigs: [[
	url: "http://forge.softwareheritage.org/source/swh-jenkins-jobs.git",
	]],
	])
	}
	}

	stage('Setup gitlab integration') {
	steps {
	script {
	setupGitlabWebhook("swh/infra/ci-cd/swh-jenkins-jobs",
	"jenkins-tools/swh-jenkins-jobs-builder",
	true, true, false)
	+ setupGitlabWebhook("swh/devel/swh-docs", "DDOC/gitlab-builds",
	+ true, true, false)

	projects = readYaml(file: 'jobs/swh-packages.yaml')
	for (project in projects) {
	if (project.containsKey("project")) {
	def jenkinsFolder = project.get('project').get('name')
	def repoName= project.get('project').get('repo_name')
	def gitlabProjectName = "swh/devel/${repoName}"
	setupGitlabWebhook(gitlabProjectName, "${jenkinsFolder}/gitlab-tests")
	setupGitlabWebhook(gitlabProjectName, "${jenkinsFolder}/gitlab-incoming-tag",
	false, false, true)
	}
	}
	}
	}
	}
	}
	}

	void setupGitlabWebhook(gitlabProjectName, jenkinsProjectName, pushEvents = true,
	mergeRequestEvents = true, tagPushEvents = false) {

	def webhookUrl = "${jenkins_url}/project/${jenkinsProjectName}"
	def gitlabProjectEncoded = java.net.URLEncoder.encode(gitlabProjectName, "UTF-8")
	def payload = """
	{
	"id": "${gitlabProjectEncoded}",
	"url": "${webhookUrl}",
	"push_events": "${pushEvents}",
	"merge_requests_events": "${mergeRequestEvents}",
	"tag_push_events": "${tagPushEvents}",
	"token": "{{jenkins_token}}"
	}
	"""

	def url = "${gitlab_url}/api/v4/projects/${gitlabProjectEncoded}/hooks"

	// get current webhooks of the gitlab project
	def response = httpRequest httpMode: 'GET', url: url,
	contentType: 'APPLICATION_JSON',
	customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]]

	def hooks = readJSON text: response.content
	for (hook in hooks) {
	if (hook.url == webhookUrl) {
	// update previously set webhook for jenkins job
	httpRequest httpMode: 'PUT', url: "${url}/${hook.id}",
	contentType: 'APPLICATION_JSON', requestBody: payload,
	customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]]
	return
	}
	}

	// add webhook for jenkins job
	httpRequest httpMode: 'POST', url: url,
	contentType: 'APPLICATION_JSON', requestBody: payload,
	customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]]

	}

	diff --git a/jobs/tools/setup-gitlab-webhooks.yaml b/jobs/tools/setup-gitlab-webhooks.yaml
	index f5eba6c..bd7f94b 100644
	--- a/jobs/tools/setup-gitlab-webhooks.yaml
	+++ b/jobs/tools/setup-gitlab-webhooks.yaml
	@@ -1,25 +1,25 @@
	- job-template:
	name: jenkins-tools/{name}
	project-type: pipeline
	description: Setup Jenkins integration for a GitLab repository
	node: built-in
	# secret jenkins token is generated when executing tox
	jenkins_token: !include-raw: jobs/templates/jenkins-token
	parameters:
	- string:
	name: gitlab_url
	description: URL of GitLab instance
	- default: https://gitlab-staging.swh.network
	+ default: "{gitlab_url}"
	- string:
	name: jenkins_url
	description: URL of Jenkins instance
	default: https://jenkins.softwareheritage.org

	dsl: !include-jinja2: setup-gitlab-webhooks.groovy.j2

	# we use a project and a job template here as we need jinja2 processing
	# to inline the generated jenkins token into the pipeline groovy script
	- project:
	name: setup-gitlab-webhooks
	jobs:
	- "jenkins-tools/{name}"

File Metadata

Mime Type: text/x-diff
Expires: Tue, Apr 15, 5:15 AM (1 h, 29 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3247178

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions