Page MenuHomeSoftware Heritage

docs: Activate build on docs diff
Closed, ResolvedPublic

Description

The sphinx-dev build is a pain to maintain.

Diff should be built to catch errors immediately.

Incidentally, the doc maintenance becomes a shared effort
and no longer a pain to keep it green after events.

Related to T3648#72333

Event Timeline

ardumont triaged this task as Normal priority.Oct 15 2021, 9:04 AM
ardumont created this task.
ardumont updated the task description. (Show Details)
ardumont updated the task description. (Show Details)

I think the main challenge here will be doing this in such a way that we don't have to do a fresh clone of swh-environment (and all associated repos) every time we build.

That's probably fine for a clean deploy build (and even then...), but that's really painful to have to wait for that when submitting diffs.

For a *dev* build, we might get away with shallow cloning all members of swh-environment, which should go faster than whatever the current mr up is doing.

I have created a new Jenkins pipeline job manually to experiment on setting up that new CI build.

I ended up with that Jenkins groovy pipeline script that seems to do the job in about 10 minutes.

swh-docs-pipeline.groovy
pipeline {
  agent {
    docker {
      reuseNode true
      image 'swh-jenkins/sphinx'
      args '--tmpfs /tmp:exec -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker -u jenkins:docker --mount type=volume,src=shared-jenkins-cachedir,dst=/home/jenkins/.cache '
    }
  }

  environment {
    PHAB_CONDUIT_URL = 'https://forge.softwareheritage.org/api/'
  }

  stages {
    stage('Checkout') {
      steps {
        withCredentials([
          string(credentialsId: 'swh-public-ci',
                 variable: 'PHAB_CONDUIT_TOKEN')]) {
          sh '''
            if [ -n "$PHID" ]; then
              echo "{
                \\\"buildTargetPHID\\\": \\\"$PHID\\\",
                \\\"artifactKey\\\": \\\"jenkins.uri\\\",
                \\\"artifactType\\\": \\\"uri\\\",
                \\\"artifactData\\\": {
                  \\\"uri\\\": \\\"${BUILD_URL}\\\",
                  \\\"name\\\": \\\"Jenkins\\\",
                  \\\"ui.external\\\": true
                }
              }" | arc call-conduit --conduit-uri $PHAB_CONDUIT_URL --conduit-token $PHAB_CONDUIT_TOKEN harbormaster.createartifact
              echo "{
                \\\"buildTargetPHID\\\": \\\"$PHID\\\",
                \\\"artifactKey\\\": \\\"jenkins.console.uri\\\",
                \\\"artifactType\\\": \\\"uri\\\",
                \\\"artifactData\\\": {
                  \\\"uri\\\": \\\"${BUILD_URL}console\\\",
                  \\\"name\\\": \\\"Jenkins console\\\",
                  \\\"ui.external\\\": true
                }
              }" | arc call-conduit --conduit-uri $PHAB_CONDUIT_URL --conduit-token $PHAB_CONDUIT_TOKEN harbormaster.createartifact
              python3 -m pyarcanist send-message work $PHID
            fi
            '''
        }
        checkout([
          $class: 'GitSCM',
          doGenerateSubmoduleConfigurations: false,
          extensions: [[$class: 'CloneOption', depth: 1, shallow: true]],
          gitTool: 'Default',
          submoduleCfg: [],
          userRemoteConfigs: [
            [url: 'https://forge.softwareheritage.org/source/swh-environment.git'],
          ],
        ])
      }
    }
    stage('Checkout swh modules') {
      steps {
        script {
          sh'''#!/bin/bash
          crudini --del .mrconfig snippets
          crudini --del .mrconfig swh-py-template
          sed -i '/swh-docs/!s/git clone/git clone --depth 1/g' .mrconfig
          mr -j 4 -t update
          '''
        }
      }
    }
    stage('Apply phabricator diff') {
      steps {
        dir('swh-docs') {
          script {
            comment = sh(
              script: '''
              #!/bin/bash
              set -e

              ORIGINAL_TAG=base-revision-${BUILD_NUMBER}-D${REV_ID}
              ORIGINAL_COMMIT=$(git rev-parse --short=10 HEAD)
              echo "==== Patch application report for D${REV_ID} (id=${DIFF_ID}) ===="
              echo
              git tag -f ${ORIGINAL_TAG} >/dev/null
              git fetch -n ${STAGING_URI} +refs/tags/phabricator/diff/${DIFF_ID}:diff-target +refs/tags/phabricator/base/${DIFF_ID}:diff-base >/dev/null
              rebased=0
              if git merge-base --is-ancestor diff-base HEAD; then
                  echo "**Rebasing** onto ${ORIGINAL_COMMIT}..."
                  echo
                  echo '```'
                  git rebase --onto ${ORIGINAL_TAG} diff-base diff-target && rebased=1 || rebase_exit=$?
                  echo '```'
                  if [ $rebased -eq 0 ]; then
                      echo
                      echo "Rebase failed (ret=${rebase_exit})!"
                      echo
                      echo '```'
                      git rebase --abort || true
                      echo '```'
                  fi
              fi

              merged=0
              if [ $rebased -eq 0 ]; then
                  echo "Could not rebase; Attempt **merge** onto ${ORIGINAL_COMMIT}..."
                  echo
                  echo '```'
                  git merge --no-edit diff-target && merged=1 || merge_exit=$?
                  echo '```'
                  if [ $merged -eq 0 ]; then
                      echo
                      echo "Merge failed (ret=${merge_exit})!"
                      echo
                  fi
              fi

              if [ $merged -eq 1 -o $rebased -eq 1 ]; then
                  echo
                  echo "===== Changes applied before test ====="
                  echo
                  echo '```'
                  git log ${ORIGINAL_TAG}..
                  echo '```'
              else
                  echo '**Attempts to apply the diff failed!**'
                  exit 1
              fi
              git tag -d ${ORIGINAL_TAG} >/dev/null
              ''',
              label: 'Apply patch',
              returnStdout: true,
            )

            writeFile(
              file: '.phabricator-comment',
              text: comment,
            )
          }
        }
      }
    }
    stage('Build Software Heritage documentation') {
      steps {
        dir('swh-docs') {
          script {
            sh '''#!/bin/bash
            git clean -dfx
            SPHINXOPTS='-W -q --keep-going -w errors.log' SPHINXOPTCOLOR='--no-color' tox -e sphinx-dev
            '''
          }
        }
      }
    }
  } // stages
  
  post {
    always {
      step([$class: 'PhabricatorNotifier',
            commentOnSuccess: true,
            commentWithConsoleLinkOnFailure: true,
            commentFile: 'swh-docs/.phabricator-comment',
            commentSize: '1000000',
            preserveFormatting: true,
            processLint: true,
            lintFile: '.phabricator-lint',
            lintFileSize: '1000000',
      ])
      
      publishHTML (target: [
        allowMissing: true,
        alwaysLinkToLastBuild: false,
        keepAll: true,
        reportDir: 'swh-docs/docs/_build/html',
        reportFiles: 'index.html',
        reportName: "SWH Documentation"
      ])

      publishHTML (target: [
        allowMissing: true,
        alwaysLinkToLastBuild: false,
        keepAll: true,
        reportDir: 'swh-docs/user/_build/html',
        reportFiles: 'index.html',
        reportName: "SWH User Documentation"
      ])
      
      publishHTML (target: [
        allowMissing: true,
        alwaysLinkToLastBuild: false,
        keepAll: true,
        reportDir: 'swh-docs/sysadm/_build/html',
        reportFiles: 'index.html',
        reportName: "SWH sysadmin Documentation"
      ])
    } // always
    cleanup {
      cleanWs()
    }
  } // post
    
} // pipeline

Next step: configure that new job in swh-jenkins-jobs repository.