diff --git a/jobs/swh-docs.yaml b/jobs/swh-docs.yaml index 5ff6e02..48a2941 100644 --- a/jobs/swh-docs.yaml +++ b/jobs/swh-docs.yaml @@ -1,156 +1,178 @@ - project: name: DDOC display-name: swh-docs + gitlab_url: https://gitlab-staging.swh.network + gitlab_connection_name: gitlab-staging jobs: - "{name}" - "{name}/publish" - "{name}/dev" - "{name}/build-on-diff" + - "{name}/gitlab-merge-requests" - job-template: name: "{name}/publish" description: Build the documentation and publish it node: swh-sphinx auth-token: ph4br1cat0r properties: - build-discarder: days-to-keep: 90 triggers: - timed: "@midnight" scm: - git: url: https://forge.softwareheritage.org/source/swh-environment.git builders: - shell: | #!/bin/bash set -ex crudini --del .mrconfig snippets crudini --del .mrconfig swh-py-template # Don't fetch potentially large debian branches sed -i '/swh-docs/!s/git clone/git clone --single-branch/g' .mrconfig mr -j 4 -t update mr -j 4 -t run sh -c 'git checkout --detach `git describe --abbrev=0 --tags`; git clean -dfx' cd swh-docs git checkout master git clean -dfx SPHINXOPTCOLOR='--no-color' tox -e sphinx publishers: - ssh: target: "devel" site: "pergamon" clean-remote: true source: "swh-docs/docs/_build/html/**" remove-prefix: "swh-docs/docs/_build/html" fail-on-error: true verbose: true - ssh: target: "user" site: "pergamon" clean-remote: true source: "swh-docs/user/_build/html/**" remove-prefix: "swh-docs/user/_build/html" fail-on-error: true verbose: true - ssh: target: "sysadm" site: "pergamon" clean-remote: true source: "swh-docs/sysadm/_build/html/**" remove-prefix: "swh-docs/sysadm/_build/html" fail-on-error: true verbose: true - job-template: name: "{name}/dev" description: Build the documentation from git repos node: swh-sphinx auth-token: ph4br1cat0r properties: - build-discarder: days-to-keep: 90 triggers: - timed: "H 9-22/2 * * *" scm: - git: url: https://forge.softwareheritage.org/source/swh-environment.git builders: - shell: | #!/bin/bash set -ex crudini --del .mrconfig snippets crudini --del .mrconfig swh-py-template # Don't fetch potentially large debian branches sed -i '/swh-docs/!s/git clone/git clone --single-branch/g' .mrconfig mr -j 4 -t update mr -t run sh -c 'echo -n "`basename $PWD` " && git describe' | grep -v 'mr run' > revisions.txt cd swh-docs git clean -dfx SPHINXOPTS='-W -q --keep-going -w errors.log' SPHINXOPTCOLOR='--no-color' tox -e sphinx-dev publishers: - archive: artifacts: "revisions.txt,swh-docs/docs/errors.log,swh-docs/.tox/log/*" - html-publisher: name: "SWH Documentation (HEAD)" dir: "swh-docs/docs/_build/html" files: "index.html" keep-all: false includes: "**/*" - html-publisher: name: "SWH User Documentation (HEAD)" dir: "swh-docs/user/_build/html" files: "index.html" keep-all: false includes: "**/*" - html-publisher: name: "SWH sysadmin Documentation (HEAD)" dir: "swh-docs/sysadm/_build/html" files: "index.html" keep-all: false includes: "**/*" -- job-template: +- job-template: &doc_build_on_diff name: "{name}/build-on-diff" display_name: Phab. diff project-type: pipeline docker_image: sphinx concurrent: true sandbox: true + gitlab_project: false auth-token: "ph4br1cat0r" properties: - build-discarder: artifact-num-to-keep: 20 days-to-keep: 90 parameters: - git-parameter: name: REVISION type: PT_REVISION defaultValue: master description: Git revision to build. - string: name: PHID description: PHID of the Phabricator target object on which results will be reported. - string: name: DIFF_ID description: ID of the Diff patch to apply, if any - string: name: REV_ID description: ID of the Phabricator revision, if any - string: name: STAGING_URI description: URI of the staging repository dsl: !include-jinja2: templates/swh-docs-pipeline-diff.groovy.j2 + +- job-template: + name: "{name}/gitlab-merge-requests" + display_name: GitLab merge requests + gitlab_project: true + auth-token: + parameters: + properties: + - gitlab: + connection: "{gitlab_connection_name}" + triggers: + - gitlab: + trigger-push: true + trigger-merge-request: true + add-ci-message: true + cancel-pending-builds-on-update: true + # secret jenkins token is generated when executing tox + secret-token: !include-raw: jobs/templates/jenkins-token diff --git a/jobs/templates/swh-docs-pipeline-diff.groovy.j2 b/jobs/templates/swh-docs-pipeline-diff.groovy.j2 index b56de42..e7c11e3 100644 --- a/jobs/templates/swh-docs-pipeline-diff.groovy.j2 +++ b/jobs/templates/swh-docs-pipeline-diff.groovy.j2 @@ -1,116 +1,142 @@ pipeline { {% filter indent(width=2) %} {%- include 'templates/includes/agent-docker.groovy.j2' -%} {% endfilter %} + {%- if not gitlab_project %} environment { PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' } + {%- endif %} stages { stage('Checkout swh environment') { steps { + {%- if not gitlab_project %} {% filter indent(width=8) %} {%- include 'templates/includes/create-phabricator-artifacts.groovy.j2' -%} {% endfilter %} + {%- endif %} checkout([ $class: 'GitSCM', doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', depth: 1, shallow: true]], gitTool: 'Default', submoduleCfg: [], - userRemoteConfigs: [ - [url: 'https://forge.softwareheritage.org/source/swh-environment.git'], - ], + userRemoteConfigs: [[ + {%- if not gitlab_project %} + url: 'https://forge.softwareheritage.org/source/swh-environment.git' + {%- else %} + url: '{{gitlab_url}}/swh/devel/swh-environment.git' + {%- endif %} + ]], ]) } } stage('Checkout swh modules') { steps { script { sh'''#!/bin/bash crudini --del .mrconfig snippets crudini --del .mrconfig swh-py-template # only fetch the HEAD branch instead of all (potentially large) branches sed -i '/swh-docs/!s/git clone/git clone --single-branch/g' .mrconfig mr -j 4 -t update ''' } } } + {%- if not gitlab_project %} stage('Apply phabricator diff') { steps { dir('swh-docs') { + {%- if not gitlab_project %} {% filter indent(width=10) %} {%- include 'templates/includes/script-apply-phabricator-diff.groovy.j2' -%} {% endfilter %} + {%- else %} + sh "git checkout ${env.gitlabSourceBranch}" + {%- endif %} } } } + {%- endif %} stage('Build Software Heritage documentation') { steps { dir('swh-docs') { script { sh '''#!/bin/bash SPHINXOPTS='-W -q --keep-going -w errors.log' SPHINXOPTCOLOR='--no-color' tox -e sphinx-dev ''' } } } } } post { always { + {%- if not gitlab_project %} step([$class: 'PhabricatorNotifier', commentOnSuccess: true, commentWithConsoleLinkOnFailure: true, commentFile: 'swh-docs/.phabricator-comment', commentSize: '1000000', preserveFormatting: true, processLint: true, lintFile: '.phabricator-lint', lintFileSize: '1000000', ]) + {%- else %} + failure { + updateGitlabCommitStatus name: 'jenkins', state: 'failed' + } + success { + updateGitlabCommitStatus name: 'jenkins', state: 'success' + } + aborted { + updateGitlabCommitStatus name: 'jenkins', state: 'canceled' + } + {%- endif %} archiveArtifacts( allowEmptyArchive: true, artifacts: 'swh-docs/docs/errors.log,swh-docs/.tox/log/*', fingerprint: true, ) publishHTML (target: [ allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true, reportDir: 'swh-docs/docs/_build/html', reportFiles: 'index.html', reportName: 'SWH Documentation' ]) publishHTML (target: [ allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true, reportDir: 'swh-docs/user/_build/html', reportFiles: 'index.html', reportName: 'SWH User Documentation' ]) publishHTML (target: [ allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true, reportDir: 'swh-docs/sysadm/_build/html', reportFiles: 'index.html', reportName: 'SWH sysadmin Documentation' ]) } cleanup { cleanWs() } } } diff --git a/jobs/tools/setup-gitlab-webhooks.groovy.j2 b/jobs/tools/setup-gitlab-webhooks.groovy.j2 index 3834e72..e22c757 100644 --- a/jobs/tools/setup-gitlab-webhooks.groovy.j2 +++ b/jobs/tools/setup-gitlab-webhooks.groovy.j2 @@ -1,82 +1,85 @@ pipeline { agent any environment { GITLAB_TOKEN = credentials("jenkins-gitlab-token") } stages { stage('Checkout Repository') { steps { checkout([ $class: 'GitSCM', branches: [[name: 'gitlab-integration']], userRemoteConfigs: [[ url: "http://forge.softwareheritage.org/source/swh-jenkins-jobs.git", ]], ]) } } stage('Setup gitlab integration') { steps { script { + setupGitlabWebhook("swh/devel/swh-docs", "DDOC/gitlab-merge-requests", + false, true, false) + projects = readYaml(file: 'jobs/swh-packages.yaml') for (project in projects) { if (project.containsKey("project")) { def jenkinsFolder = project.get('project').get('name') def repoName= project.get('project').get('repo_name') def gitlabProjectName = "swh/devel/${repoName}" setupGitlabWebhook(gitlabProjectName, "${jenkinsFolder}/gitlab-tests") setupGitlabWebhook(gitlabProjectName, "${jenkinsFolder}/gitlab-incoming-tag", false, false, true) } } } } } } } void setupGitlabWebhook(gitlabProjectName, jenkinsProjectName, pushEvents = true, mergeRequestEvents = true, tagPushEvents = false) { def webhookUrl = "${jenkins_url}/project/${jenkinsProjectName}" def gitlabProjectEncoded = java.net.URLEncoder.encode(gitlabProjectName, "UTF-8") def payload = """ { "id": "${gitlabProjectEncoded}", "url": "${webhookUrl}", "push_events": "${pushEvents}", "merge_requests_events": "${mergeRequestEvents}", "tag_push_events": "${tagPushEvents}", "token": "{{jenkins_token}}" } """ def url = "${gitlab_url}/api/v4/projects/${gitlabProjectEncoded}/hooks" // get current webhooks of the gitlab project def response = httpRequest httpMode: 'GET', url: url, contentType: 'APPLICATION_JSON', customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]] def hooks = readJSON text: response.content for (hook in hooks) { if (hook.url == webhookUrl) { // update previously set webhook for jenkins job httpRequest httpMode: 'PUT', url: "${url}/${hook.id}", contentType: 'APPLICATION_JSON', requestBody: payload, customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]] return } } // add webhook for jenkins job httpRequest httpMode: 'POST', url: url, contentType: 'APPLICATION_JSON', requestBody: payload, customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]] }