diff --git a/jobs/swh-docs.yaml b/jobs/swh-docs.yaml --- a/jobs/swh-docs.yaml +++ b/jobs/swh-docs.yaml @@ -2,11 +2,12 @@ name: DDOC display-name: swh-docs jobs: - - "{name}" + - "{name}/publish" + - "{name}/dev" + - "{name}/build-on-diff" - -- job: - name: DDOC/publish +- job-template: + name: "{name}/publish" description: Build the documentation and publish it node: swh-sphinx auth-token: ph4br1cat0r @@ -30,33 +31,32 @@ publishers: - ssh: - target: 'devel' - site: 'pergamon' + target: "devel" + site: "pergamon" clean-remote: true - source: 'swh-docs/docs/_build/html/**' - remove-prefix: 'swh-docs/docs/_build/html' + source: "swh-docs/docs/_build/html/**" + remove-prefix: "swh-docs/docs/_build/html" fail-on-error: true verbose: true - ssh: - target: 'user' - site: 'pergamon' + target: "user" + site: "pergamon" clean-remote: true - source: 'swh-docs/user/_build/html/**' - remove-prefix: 'swh-docs/user/_build/html' + source: "swh-docs/user/_build/html/**" + remove-prefix: "swh-docs/user/_build/html" fail-on-error: true verbose: true - ssh: - target: 'sysadm' - site: 'pergamon' + target: "sysadm" + site: "pergamon" clean-remote: true - source: 'swh-docs/sysadm/_build/html/**' - remove-prefix: 'swh-docs/sysadm/_build/html' + source: "swh-docs/sysadm/_build/html/**" + remove-prefix: "swh-docs/sysadm/_build/html" fail-on-error: true verbose: true - -- job: - name: DDOC/dev +- job-template: + name: "{name}/dev" description: Build the documentation from git repos node: swh-sphinx auth-token: ph4br1cat0r @@ -83,20 +83,52 @@ - archive: artifacts: "revisions.txt,swh-docs/docs/errors.log,swh-docs/.tox/log/*" - html-publisher: - name: 'SWH Documentation (HEAD)' - dir: 'swh-docs/docs/_build/html' - files: 'index.html' + name: "SWH Documentation (HEAD)" + dir: "swh-docs/docs/_build/html" + files: "index.html" keep-all: false - includes: '**/*' + includes: "**/*" - html-publisher: - name: 'SWH User Documentation (HEAD)' - dir: 'swh-docs/user/_build/html' - files: 'index.html' + name: "SWH User Documentation (HEAD)" + dir: "swh-docs/user/_build/html" + files: "index.html" keep-all: false - includes: '**/*' + includes: "**/*" - html-publisher: - name: 'SWH sysadmin Documentation (HEAD)' - dir: 'swh-docs/sysadm/_build/html' - files: 'index.html' + name: "SWH sysadmin Documentation (HEAD)" + dir: "swh-docs/sysadm/_build/html" + files: "index.html" keep-all: false - includes: '**/*' + includes: "**/*" + +- job-template: + name: "{name}/build-on-diff" + display_name: Phab. diff + project-type: pipeline + docker_image: sphinx + concurrent: true + sandbox: true + auth-token: "ph4br1cat0r" + properties: + - build-discarder: + artifact-num-to-keep: 20 + parameters: + - git-parameter: + name: REVISION + type: PT_REVISION + defaultValue: master + description: Git revision to build. + - string: + name: PHID + description: PHID of the Phabricator target object on which results will be reported. + - string: + name: DIFF_ID + description: ID of the Diff patch to apply, if any + - string: + name: REV_ID + description: ID of the Phabricator revision, if any + - string: + name: STAGING_URI + description: URI of the staging repository + + dsl: !include-jinja2: templates/swh-docs-pipeline-diff.groovy.j2 diff --git a/jobs/templates/includes/script-apply-phabricator-diff.groovy.j2 b/jobs/templates/includes/script-apply-phabricator-diff.groovy.j2 new file mode 100644 --- /dev/null +++ b/jobs/templates/includes/script-apply-phabricator-diff.groovy.j2 @@ -0,0 +1,65 @@ +script { + comment = sh( + script: ''' + #!/bin/bash + set -e + + ORIGINAL_TAG=base-revision-${BUILD_NUMBER}-D${REV_ID} + ORIGINAL_COMMIT=$(git rev-parse --short=10 HEAD) + echo "==== Patch application report for D${REV_ID} (id=${DIFF_ID}) ====" + echo + git tag -f ${ORIGINAL_TAG} >/dev/null + git fetch -n ${STAGING_URI} +refs/tags/phabricator/diff/${DIFF_ID}:diff-target +refs/tags/phabricator/base/${DIFF_ID}:diff-base >/dev/null + rebased=0 + if git merge-base --is-ancestor diff-base HEAD; then + echo "**Rebasing** onto ${ORIGINAL_COMMIT}..." + echo + echo '```' + git rebase --onto ${ORIGINAL_TAG} diff-base diff-target && rebased=1 || rebase_exit=$? + echo '```' + if [ $rebased -eq 0 ]; then + echo + echo "Rebase failed (ret=${rebase_exit})!" + echo + echo '```' + git rebase --abort || true + echo '```' + fi + fi + + merged=0 + if [ $rebased -eq 0 ]; then + echo "Could not rebase; Attempt **merge** onto ${ORIGINAL_COMMIT}..." + echo + echo '```' + git merge --no-edit diff-target && merged=1 || merge_exit=$? + echo '```' + if [ $merged -eq 0 ]; then + echo + echo "Merge failed (ret=${merge_exit})!" + echo + fi + fi + + if [ $merged -eq 1 -o $rebased -eq 1 ]; then + echo + echo "===== Changes applied before test =====" + echo + echo '```' + git log ${ORIGINAL_TAG}.. + echo '```' + else + echo '**Attempts to apply the diff failed!**' + exit 1 + fi + git tag -d ${ORIGINAL_TAG} >/dev/null + ''', + label: 'Apply patch', + returnStdout: true, + ) + + writeFile( + file: '.phabricator-comment', + text: comment, + ) +} diff --git a/jobs/templates/includes/stage-apply-phabricator-diff.groovy.j2 b/jobs/templates/includes/stage-apply-phabricator-diff.groovy.j2 deleted file mode 100644 --- a/jobs/templates/includes/stage-apply-phabricator-diff.groovy.j2 +++ /dev/null @@ -1,69 +0,0 @@ -stage('Apply phabricator diff') { - steps { - script { - comment = sh( - script: ''' - #!/bin/bash - set -e - - ORIGINAL_TAG=base-revision-${BUILD_NUMBER}-D${REV_ID} - ORIGINAL_COMMIT=$(git rev-parse --short=10 HEAD) - echo "==== Patch application report for D${REV_ID} (id=${DIFF_ID}) ====" - echo - git tag -f ${ORIGINAL_TAG} >/dev/null - git fetch -n ${STAGING_URI} +refs/tags/phabricator/diff/${DIFF_ID}:diff-target +refs/tags/phabricator/base/${DIFF_ID}:diff-base >/dev/null - rebased=0 - if git merge-base --is-ancestor diff-base HEAD; then - echo "**Rebasing** onto ${ORIGINAL_COMMIT}..." - echo - echo '```' - git rebase --onto ${ORIGINAL_TAG} diff-base diff-target && rebased=1 || rebase_exit=$? - echo '```' - if [ $rebased -eq 0 ]; then - echo - echo "Rebase failed (ret=${rebase_exit})!" - echo - echo '```' - git rebase --abort || true - echo '```' - fi - fi - - merged=0 - if [ $rebased -eq 0 ]; then - echo "Could not rebase; Attempt **merge** onto ${ORIGINAL_COMMIT}..." - echo - echo '```' - git merge --no-edit diff-target && merged=1 || merge_exit=$? - echo '```' - if [ $merged -eq 0 ]; then - echo - echo "Merge failed (ret=${merge_exit})!" - echo - fi - fi - - if [ $merged -eq 1 -o $rebased -eq 1 ]; then - echo - echo "===== Changes applied before test =====" - echo - echo '```' - git log ${ORIGINAL_TAG}.. - echo '```' - else - echo '**Attempts to apply the diff failed!**' - exit 1 - fi - git tag -d ${ORIGINAL_TAG} >/dev/null - ''', - label: 'Apply patch', - returnStdout: true, - ) - - writeFile( - file: '.phabricator-comment', - text: comment, - ) - } - } -} diff --git a/jobs/templates/swh-docs-pipeline-diff.groovy.j2 b/jobs/templates/swh-docs-pipeline-diff.groovy.j2 new file mode 100644 --- /dev/null +++ b/jobs/templates/swh-docs-pipeline-diff.groovy.j2 @@ -0,0 +1,116 @@ +pipeline { + {% filter indent(width=2) %} + {%- include 'templates/includes/agent-docker.groovy.j2' -%} + {% endfilter %} + + environment { + PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' + } + + stages { + stage('Checkout swh environment') { + steps { + {% filter indent(width=8) %} + {%- include 'templates/includes/create-phabricator-artifacts.groovy.j2' -%} + {% endfilter %} + checkout([ + $class: 'GitSCM', + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'CloneOption', depth: 1, shallow: true]], + gitTool: 'Default', + submoduleCfg: [], + userRemoteConfigs: [ + [url: 'https://forge.softwareheritage.org/source/swh-environment.git'], + ], + ]) + } + } + + stage('Checkout swh modules') { + steps { + script { + sh'''#!/bin/bash + crudini --del .mrconfig snippets + crudini --del .mrconfig swh-py-template + # force shallow clone of swh-repos except swh-docs + sed -i '/swh-docs/!s/git clone/git clone --depth 1/g' .mrconfig + mr -j 4 -t update + ''' + } + } + } + + stage('Apply phabricator diff') { + steps { + dir('swh-docs') { + {% filter indent(width=10) %} + {%- include 'templates/includes/script-apply-phabricator-diff.groovy.j2' -%} + {% endfilter %} + } + } + } + + stage('Build Software Heritage documentation') { + steps { + dir('swh-docs') { + script { + sh '''#!/bin/bash + SPHINXOPTS='-W -q --keep-going -w errors.log' SPHINXOPTCOLOR='--no-color' tox -e sphinx-dev + ''' + } + } + } + } + } + + post { + always { + step([$class: 'PhabricatorNotifier', + commentOnSuccess: true, + commentWithConsoleLinkOnFailure: true, + commentFile: 'swh-docs/.phabricator-comment', + commentSize: '1000000', + preserveFormatting: true, + processLint: true, + lintFile: '.phabricator-lint', + lintFileSize: '1000000', + ]) + + archiveArtifacts( + allowEmptyArchive: true, + artifacts: 'swh-docs/docs/errors.log,swh-docs/.tox/log/*', + fingerprint: true, + ) + + publishHTML (target: [ + allowMissing: true, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'swh-docs/docs/_build/html', + reportFiles: 'index.html', + reportName: 'SWH Documentation' + ]) + + publishHTML (target: [ + allowMissing: true, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'swh-docs/user/_build/html', + reportFiles: 'index.html', + reportName: 'SWH User Documentation' + ]) + + publishHTML (target: [ + allowMissing: true, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'swh-docs/sysadm/_build/html', + reportFiles: 'index.html', + reportName: 'SWH sysadmin Documentation' + ]) + } + cleanup { + cleanWs() + } + } +} \ No newline at end of file diff --git a/jobs/templates/swh-pipeline.groovy.j2 b/jobs/templates/swh-pipeline.groovy.j2 --- a/jobs/templates/swh-pipeline.groovy.j2 +++ b/jobs/templates/swh-pipeline.groovy.j2 @@ -29,9 +29,13 @@ } {%- if phabricator_diff %} - {% filter indent(width=4) %} - {%- include 'includes/stage-apply-phabricator-diff.groovy.j2' -%} - {% endfilter %} + stage('Apply phabricator diff') { + steps { + {% filter indent(width=8) %} + {%- include 'includes/script-apply-phabricator-diff.groovy.j2' -%} + {% endfilter %} + } + } {%- endif %} stage('Static analysis') {