diff --git a/jobs/swh-docs.yaml b/jobs/swh-docs.yaml index d19e1ec..91ee3ad 100644 --- a/jobs/swh-docs.yaml +++ b/jobs/swh-docs.yaml @@ -1,102 +1,134 @@ - project: name: DDOC display-name: swh-docs jobs: - - "{name}" + - "{name}/publish" + - "{name}/dev" + - "{name}/build-on-diff" - -- job: - name: DDOC/publish +- job-template: + name: "{name}/publish" description: Build the documentation and publish it node: swh-sphinx auth-token: ph4br1cat0r triggers: - timed: "@midnight" scm: - git: url: https://forge.softwareheritage.org/source/swh-environment.git builders: - shell: | #!/bin/bash mr -j 4 -t update mr -j 4 -t run sh -c 'git checkout --detach `git describe --abbrev=0 --tags`; git clean -dfx' cd swh-docs git checkout master git clean -dfx SPHINXOPTCOLOR='--no-color' tox -e sphinx publishers: - ssh: - target: 'devel' - site: 'pergamon' + target: "devel" + site: "pergamon" clean-remote: true - source: 'swh-docs/docs/_build/html/**' - remove-prefix: 'swh-docs/docs/_build/html' + source: "swh-docs/docs/_build/html/**" + remove-prefix: "swh-docs/docs/_build/html" fail-on-error: true verbose: true - ssh: - target: 'user' - site: 'pergamon' + target: "user" + site: "pergamon" clean-remote: true - source: 'swh-docs/user/_build/html/**' - remove-prefix: 'swh-docs/user/_build/html' + source: "swh-docs/user/_build/html/**" + remove-prefix: "swh-docs/user/_build/html" fail-on-error: true verbose: true - ssh: - target: 'sysadm' - site: 'pergamon' + target: "sysadm" + site: "pergamon" clean-remote: true - source: 'swh-docs/sysadm/_build/html/**' - remove-prefix: 'swh-docs/sysadm/_build/html' + source: "swh-docs/sysadm/_build/html/**" + remove-prefix: "swh-docs/sysadm/_build/html" fail-on-error: true verbose: true - -- job: - name: DDOC/dev +- job-template: + name: "{name}/dev" description: Build the documentation from git repos node: swh-sphinx auth-token: ph4br1cat0r triggers: - timed: "H 9-22/2 * * *" scm: - git: url: https://forge.softwareheritage.org/source/swh-environment.git builders: - shell: | #!/bin/bash crudini --del .mrconfig snippets crudini --del .mrconfig swh-py-template mr -j 4 -t update mr -t run sh -c 'echo -n "`basename $PWD` " && git describe' | grep -v 'mr run' > revisions.txt cd swh-docs git clean -dfx SPHINXOPTS='-W -q --keep-going -w errors.log' SPHINXOPTCOLOR='--no-color' tox -e sphinx-dev publishers: - archive: artifacts: "revisions.txt,swh-docs/docs/errors.log,swh-docs/.tox/log/*" - html-publisher: - name: 'SWH Documentation (HEAD)' - dir: 'swh-docs/docs/_build/html' - files: 'index.html' + name: "SWH Documentation (HEAD)" + dir: "swh-docs/docs/_build/html" + files: "index.html" keep-all: false - includes: '**/*' + includes: "**/*" - html-publisher: - name: 'SWH User Documentation (HEAD)' - dir: 'swh-docs/user/_build/html' - files: 'index.html' + name: "SWH User Documentation (HEAD)" + dir: "swh-docs/user/_build/html" + files: "index.html" keep-all: false - includes: '**/*' + includes: "**/*" - html-publisher: - name: 'SWH sysadmin Documentation (HEAD)' - dir: 'swh-docs/sysadm/_build/html' - files: 'index.html' + name: "SWH sysadmin Documentation (HEAD)" + dir: "swh-docs/sysadm/_build/html" + files: "index.html" keep-all: false - includes: '**/*' + includes: "**/*" + +- job-template: + name: "{name}/build-on-diff" + display_name: Phab. diff + project-type: pipeline + docker_image: sphinx + concurrent: true + sandbox: true + auth-token: "ph4br1cat0r" + properties: + - build-discarder: + artifact-num-to-keep: 20 + parameters: + - git-parameter: + name: REVISION + type: PT_REVISION + defaultValue: master + description: Git revision to build. + - string: + name: PHID + description: PHID of the Phabricator target object on which results will be reported. + - string: + name: DIFF_ID + description: ID of the Diff patch to apply, if any + - string: + name: REV_ID + description: ID of the Phabricator revision, if any + - string: + name: STAGING_URI + description: URI of the staging repository + + dsl: !include-jinja2: templates/swh-docs-pipeline-diff.groovy.j2 diff --git a/jobs/templates/includes/script-apply-phabricator-diff.groovy.j2 b/jobs/templates/includes/script-apply-phabricator-diff.groovy.j2 new file mode 100644 index 0000000..0569e3d --- /dev/null +++ b/jobs/templates/includes/script-apply-phabricator-diff.groovy.j2 @@ -0,0 +1,65 @@ +script { + comment = sh( + script: ''' + #!/bin/bash + set -e + + ORIGINAL_TAG=base-revision-${BUILD_NUMBER}-D${REV_ID} + ORIGINAL_COMMIT=$(git rev-parse --short=10 HEAD) + echo "==== Patch application report for D${REV_ID} (id=${DIFF_ID}) ====" + echo + git tag -f ${ORIGINAL_TAG} >/dev/null + git fetch -n ${STAGING_URI} +refs/tags/phabricator/diff/${DIFF_ID}:diff-target +refs/tags/phabricator/base/${DIFF_ID}:diff-base >/dev/null + rebased=0 + if git merge-base --is-ancestor diff-base HEAD; then + echo "**Rebasing** onto ${ORIGINAL_COMMIT}..." + echo + echo '```' + git rebase --onto ${ORIGINAL_TAG} diff-base diff-target && rebased=1 || rebase_exit=$? + echo '```' + if [ $rebased -eq 0 ]; then + echo + echo "Rebase failed (ret=${rebase_exit})!" + echo + echo '```' + git rebase --abort || true + echo '```' + fi + fi + + merged=0 + if [ $rebased -eq 0 ]; then + echo "Could not rebase; Attempt **merge** onto ${ORIGINAL_COMMIT}..." + echo + echo '```' + git merge --no-edit diff-target && merged=1 || merge_exit=$? + echo '```' + if [ $merged -eq 0 ]; then + echo + echo "Merge failed (ret=${merge_exit})!" + echo + fi + fi + + if [ $merged -eq 1 -o $rebased -eq 1 ]; then + echo + echo "===== Changes applied before test =====" + echo + echo '```' + git log ${ORIGINAL_TAG}.. + echo '```' + else + echo '**Attempts to apply the diff failed!**' + exit 1 + fi + git tag -d ${ORIGINAL_TAG} >/dev/null + ''', + label: 'Apply patch', + returnStdout: true, + ) + + writeFile( + file: '.phabricator-comment', + text: comment, + ) +} diff --git a/jobs/templates/includes/stage-apply-phabricator-diff.groovy.j2 b/jobs/templates/includes/stage-apply-phabricator-diff.groovy.j2 deleted file mode 100644 index 413eae9..0000000 --- a/jobs/templates/includes/stage-apply-phabricator-diff.groovy.j2 +++ /dev/null @@ -1,69 +0,0 @@ -stage('Apply phabricator diff') { - steps { - script { - comment = sh( - script: ''' - #!/bin/bash - set -e - - ORIGINAL_TAG=base-revision-${BUILD_NUMBER}-D${REV_ID} - ORIGINAL_COMMIT=$(git rev-parse --short=10 HEAD) - echo "==== Patch application report for D${REV_ID} (id=${DIFF_ID}) ====" - echo - git tag -f ${ORIGINAL_TAG} >/dev/null - git fetch -n ${STAGING_URI} +refs/tags/phabricator/diff/${DIFF_ID}:diff-target +refs/tags/phabricator/base/${DIFF_ID}:diff-base >/dev/null - rebased=0 - if git merge-base --is-ancestor diff-base HEAD; then - echo "**Rebasing** onto ${ORIGINAL_COMMIT}..." - echo - echo '```' - git rebase --onto ${ORIGINAL_TAG} diff-base diff-target && rebased=1 || rebase_exit=$? - echo '```' - if [ $rebased -eq 0 ]; then - echo - echo "Rebase failed (ret=${rebase_exit})!" - echo - echo '```' - git rebase --abort || true - echo '```' - fi - fi - - merged=0 - if [ $rebased -eq 0 ]; then - echo "Could not rebase; Attempt **merge** onto ${ORIGINAL_COMMIT}..." - echo - echo '```' - git merge --no-edit diff-target && merged=1 || merge_exit=$? - echo '```' - if [ $merged -eq 0 ]; then - echo - echo "Merge failed (ret=${merge_exit})!" - echo - fi - fi - - if [ $merged -eq 1 -o $rebased -eq 1 ]; then - echo - echo "===== Changes applied before test =====" - echo - echo '```' - git log ${ORIGINAL_TAG}.. - echo '```' - else - echo '**Attempts to apply the diff failed!**' - exit 1 - fi - git tag -d ${ORIGINAL_TAG} >/dev/null - ''', - label: 'Apply patch', - returnStdout: true, - ) - - writeFile( - file: '.phabricator-comment', - text: comment, - ) - } - } -} diff --git a/jobs/templates/swh-docs-pipeline-diff.groovy.j2 b/jobs/templates/swh-docs-pipeline-diff.groovy.j2 new file mode 100644 index 0000000..711fc5d --- /dev/null +++ b/jobs/templates/swh-docs-pipeline-diff.groovy.j2 @@ -0,0 +1,116 @@ +pipeline { + {% filter indent(width=2) %} + {%- include 'templates/includes/agent-docker.groovy.j2' -%} + {% endfilter %} + + environment { + PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' + } + + stages { + stage('Checkout swh environment') { + steps { + {% filter indent(width=8) %} + {%- include 'templates/includes/create-phabricator-artifacts.groovy.j2' -%} + {% endfilter %} + checkout([ + $class: 'GitSCM', + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'CloneOption', depth: 1, shallow: true]], + gitTool: 'Default', + submoduleCfg: [], + userRemoteConfigs: [ + [url: 'https://forge.softwareheritage.org/source/swh-environment.git'], + ], + ]) + } + } + + stage('Checkout swh modules') { + steps { + script { + sh'''#!/bin/bash + crudini --del .mrconfig snippets + crudini --del .mrconfig swh-py-template + # force shallow clone of swh-repos except swh-docs + sed -i '/swh-docs/!s/git clone/git clone --depth 1/g' .mrconfig + mr -j 4 -t update + ''' + } + } + } + + stage('Apply phabricator diff') { + steps { + dir('swh-docs') { + {% filter indent(width=10) %} + {%- include 'templates/includes/script-apply-phabricator-diff.groovy.j2' -%} + {% endfilter %} + } + } + } + + stage('Build Software Heritage documentation') { + steps { + dir('swh-docs') { + script { + sh '''#!/bin/bash + SPHINXOPTS='-W -q --keep-going -w errors.log' SPHINXOPTCOLOR='--no-color' tox -e sphinx-dev + ''' + } + } + } + } + } + + post { + always { + step([$class: 'PhabricatorNotifier', + commentOnSuccess: true, + commentWithConsoleLinkOnFailure: true, + commentFile: 'swh-docs/.phabricator-comment', + commentSize: '1000000', + preserveFormatting: true, + processLint: true, + lintFile: '.phabricator-lint', + lintFileSize: '1000000', + ]) + + archiveArtifacts( + allowEmptyArchive: true, + artifacts: 'swh-docs/docs/errors.log,swh-docs/.tox/log/*', + fingerprint: true, + ) + + publishHTML (target: [ + allowMissing: true, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'swh-docs/docs/_build/html', + reportFiles: 'index.html', + reportName: 'SWH Documentation' + ]) + + publishHTML (target: [ + allowMissing: true, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'swh-docs/user/_build/html', + reportFiles: 'index.html', + reportName: 'SWH User Documentation' + ]) + + publishHTML (target: [ + allowMissing: true, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'swh-docs/sysadm/_build/html', + reportFiles: 'index.html', + reportName: 'SWH sysadmin Documentation' + ]) + } + cleanup { + cleanWs() + } + } +} \ No newline at end of file diff --git a/jobs/templates/swh-pipeline.groovy.j2 b/jobs/templates/swh-pipeline.groovy.j2 index 801fac6..ded6013 100644 --- a/jobs/templates/swh-pipeline.groovy.j2 +++ b/jobs/templates/swh-pipeline.groovy.j2 @@ -1,138 +1,142 @@ pipeline { {% filter indent(width=2) %} {%- include 'includes/agent-docker.groovy.j2' -%} {% endfilter %} environment { PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/' } stages { stage('Checkout') { steps { {% filter indent(width=8) %} {%- include 'includes/create-phabricator-artifacts.groovy.j2' -%} {% endfilter %} checkout([ $class: 'GitSCM', branches: [[name: "${params.REVISION}"]], doGenerateSubmoduleConfigurations: false, extensions: [], gitTool: 'Default', submoduleCfg: [], userRemoteConfigs: [ [url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git'], ], browser: [$class: 'Phabricator', repoUrl: 'https://forge.softwareheritage.org', repo: '{{name}}'] ]) } } {%- if phabricator_diff %} - {% filter indent(width=4) %} - {%- include 'includes/stage-apply-phabricator-diff.groovy.j2' -%} - {% endfilter %} + stage('Apply phabricator diff') { + steps { + {% filter indent(width=8) %} + {%- include 'includes/script-apply-phabricator-diff.groovy.j2' -%} + {% endfilter %} + } + } {%- endif %} stage('Static analysis') { parallel { stage ('flake8') { steps { sh '''python3 -m tox -e flake8''' } } stage ('mypy') { steps { sh '''python3 -m tox -e mypy''' } } stage ('radon') { steps { sh ''' mkdir -p reports python3 -m radon raw --json swh/ > reports/raw_report.json python3 -m radon cc --json swh/ > reports/cc_report.json python3 -m radon mi --json swh/ > reports/mi_report.json python3 -m radon hal --json swh/ > reports/hal_report.json python3 -m radon cc --xml swh/ > reports/cc_report.xml ''' } post { always { // Archive a few report files archiveArtifacts( allowEmptyArchive: true, artifacts: 'reports/*', fingerprint: true, ) // Warnings NG recordIssues( enabledForFailure: true, tools: [ ccm(pattern: '**/reports/cc_report.xml'), ], ) } } } } } // static analysis stage('Tests') { options { timeout(time: 20, unit: 'MINUTES') } parallel { {% filter indent(width=8) %} {%- include 'includes/stage-python-tests.groovy.j2' -%} {% endfilter %} {%- if do_cypress %} {% filter indent(width=8) %} {%- include 'includes/stage-cypress-tests.groovy.j2' -%} {% endfilter %} {%- endif %} stage('Sphinx documentation') { {% filter indent(width=6) %} {%- include 'includes/agent-docker-sphinx.groovy.j2' -%} {% endfilter %} steps { sh ''' if tox -a | grep -x sphinx >/dev/null then tox -e sphinx else echo WARNING: no sphinx environment in tox.ini fi ''' } } // sphinx doc } // parallel } // Tests stage } // stages post { always { step([$class: 'PhabricatorNotifier', commentOnSuccess: true, commentWithConsoleLinkOnFailure: true, commentFile: '.phabricator-comment', commentSize: '1000000', preserveFormatting: true, processLint: true, lintFile: '.phabricator-lint', lintFileSize: '1000000', ]) } // always cleanup { cleanWs() } } // post } // pipeline