diff --git a/jobs/swh-packages.yaml b/jobs/swh-packages.yaml index 5d88171..037a7df 100644 --- a/jobs/swh-packages.yaml +++ b/jobs/swh-packages.yaml @@ -1,349 +1,351 @@ - job-group: name: "swh-jobs-{name}" gitlab_url: https://gitlab-staging.swh.network gitlab_connection_name: gitlab-staging gitlab_project_name: "swh/devel/{repo_name}" jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tests-on-diff" - "{name}/gitlab-tests" + - "{name}/gitlab-incoming-tag" + - "{name}/gitlab-pypi-upload" - "debian/packages/{name}" - "debian/packages/{name}/update-for-release" - "debian/packages/{name}/automatic-backport" - "debian/packages/{name}/gbp-buildpackage" - project: name: DAUTH display-name: swh-auth repo_name: swh-auth pkg: auth python_module: swh.auth jobs: - "swh-jobs-{name}" - project: name: DCORE display-name: swh-core repo_name: swh-core pkg: core python_module: swh.core jobs: - "swh-jobs-{name}" - project: name: DCNT display-name: swh-counters repo_name: swh-counters pkg: counters python_module: swh.counters jobs: - "swh-jobs-{name}" - project: name: DDATASET display-name: swh-dataset repo_name: swh-dataset pkg: dataset python_module: swh.dataset jobs: - "swh-jobs-{name}" - project: name: DDEP display-name: swh-deposit repo_name: swh-deposit pkg: deposit python_module: swh.deposit jobs: - "swh-jobs-{name}" - project: name: DGRPH display-name: swh-graph repo_name: swh-graph pkg: graph python_module: swh.graph jobs: - "swh-jobs-{name}" - project: name: DGQL display-name: swh-graphql repo_name: swh-graphql pkg: graphql python_module: swh.graphql jobs: - "swh-jobs-{name}" - project: name: DCIDX display-name: swh-indexer repo_name: swh-indexer pkg: indexer python_module: swh.indexer jobs: - "swh-jobs-{name}" - project: name: DICP display-name: swh-icinga-plugins repo_name: swh-icinga-plugins pkg: icinga-plugins python_module: swh.icinga_plugins jobs: - "swh-jobs-{name}" - project: name: DJNL display-name: swh-journal repo_name: swh-journal docker_image: kafka pkg: journal python_module: swh.journal jobs: - "swh-jobs-{name}" - project: name: DLS display-name: swh-lister repo_name: swh-lister pkg: lister python_module: swh.lister jobs: - "swh-jobs-{name}" - project: name: DLDBZR display-name: swh-loader-bzr repo_name: swh-loader-bzr pkg: loader.bzr python_module: swh.loader.bzr jobs: - "swh-jobs-{name}" - project: name: DLDBASE display-name: swh-loader-core repo_name: swh-loader-core pkg: loader.core python_module: swh.loader.core jobs: - "swh-jobs-{name}" - project: name: DLDCVS display-name: swh-loader-cvs repo_name: swh-loader-cvs pkg: loader.cvs python_module: swh.loader.cvs include_bdist: false jobs: - "swh-jobs-{name}" - project: name: DLDG display-name: swh-loader-git repo_name: swh-loader-git pkg: loader.git python_module: swh.loader.git jobs: - "swh-jobs-{name}" - project: name: DLDMD display-name: swh-loader-metadata repo_name: swh-loader-metadata pkg: loader.metadata python_module: swh.loader.metadata jobs: - "swh-jobs-{name}" - project: name: DLDHG display-name: swh-loader-mercurial repo_name: swh-loader-mercurial pkg: loader.mercurial python_module: swh.loader.mercurial jobs: - "swh-jobs-{name}" - project: name: DLDSVN display-name: swh-loader-svn repo_name: swh-loader-svn pkg: loader.svn python_module: swh.loader.svn jobs: - "swh-jobs-{name}" - project: name: DMOD display-name: swh-model repo_name: swh-model pkg: model python_module: swh.model jobs: - "swh-jobs-{name}" - project: name: DMFCD display-name: swh-clearlydefined repo_name: swh-clearlydefined pkg: clearlydefined python_module: swh.clearlydefined jobs: - "swh-jobs-{name}" - project: name: DOBJS display-name: swh-objstorage repo_name: swh-objstorage pkg: objstorage deb-extra-repositories: ceph python_module: swh.objstorage jobs: - "swh-jobs-{name}" - project: name: DOBJSRPL display-name: swh-objstorage-replayer repo_name: swh-objstorage-replayer pkg: objstorage.replayer python_module: swh.objstorage.replayer jobs: - "swh-jobs-{name}" - project: name: DOPH display-name: swh-perfecthash repo_name: swh-perfecthash pkg: perfecthash python_module: swh.perfecthash include_bdist: false jobs: - "swh-jobs-{name}" - project: name: DPROV display-name: swh-provenance repo_name: swh-provenance pkg: provenance python_module: swh.provenance timeout: 15 jobs: - "swh-jobs-{name}" - project: name: DSEA display-name: swh-search repo_name: swh-search pkg: search deb-extra-repositories: elasticsearch python_module: swh.search jobs: - "swh-jobs-{name}" - project: name: TSBX display-name: swh-sandbox repo_name: swh-sandbox pkg: sandbox incoming_tag_auto_pypi_host: test.pypi.org python_module: swh.sandbox jobs: - "{name}" - "{name}/incoming-tag" - "{name}/pypi-upload" - "{name}/tests" - "{name}/tests-on-diff" - project: name: DTSCN display-name: swh-scanner repo_name: swh-scanner pkg: scanner python_module: swh.scanner jobs: - "swh-jobs-{name}" - project: name: DSCH display-name: swh-scheduler repo_name: swh-scheduler pkg: scheduler python_module: swh.scheduler jobs: - "swh-jobs-{name}" - project: name: DSCRUB display-name: swh-scrubber repo_name: swh-scrubber pkg: scrubber python_module: swh.scrubber jobs: - "swh-jobs-{name}" - project: name: DSTO display-name: swh-storage repo_name: swh-storage pkg: storage deb-extra-repositories: cassandra python_module: swh.storage timeout: 25 jobs: - "swh-jobs-{name}" - project: name: DVAU display-name: swh-vault repo_name: swh-vault pkg: vault python_module: swh.vault jobs: - "swh-jobs-{name}" - project: name: DWAPPS display-name: swh-web repo_name: swh-web pkg: web python_module: swh.web do_cypress: true nb_cypress_runners: 4 timeout: 30 max_concurrent: 3 jobs: - "swh-jobs-{name}" - project: name: DWCLI display-name: swh-web-client repo_name: swh-web-client pkg: web-client python_module: swh.web.client jobs: - "swh-jobs-{name}" - project: name: DFUSE display-name: swh-fuse repo_name: swh-fuse pkg: fuse python_module: swh.fuse docker_options: --privileged --device /dev/fuse jobs: - "swh-jobs-{name}" - project: name: DTPL display-name: swh-py-template repo_name: swh-py-template pkg: py-template python_module: swh.foo jobs: - "swh-jobs-{name}" diff --git a/jobs/templates/incoming-tag.groovy.j2 b/jobs/templates/incoming-tag.groovy.j2 index 7254d21..6fee916 100644 --- a/jobs/templates/incoming-tag.groovy.j2 +++ b/jobs/templates/incoming-tag.groovy.j2 @@ -1,71 +1,102 @@ pipeline { agent none stages { stage('Refresh tag list') { agent any steps { checkout([ $class: 'GitSCM', + {%- if gitlab_project %} + userRemoteConfigs: [[ + name:'origin', url: '{{gitlab_url}}/{{gitlab_project_name}}.git', + refspec: '+refs/tags/*:refs/remotes/origin/tags*' + ]], + branches: [[ + name: "${env.gitlabSourceBranch}" + ]], + browser: [ + $class: 'GitLab', + repoUrl: '{{gitlab_url}}/{{gitlab_project_name}}' + ], + extensions: [[$class: 'CloneOption', honorRefspec: true]], + {%- else %} userRemoteConfigs: [[ url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git', ]], branches: [[ name: params.GIT_TAG, ]], browser: [ $class: 'Phabricator', repo: '{{repo_name}}', repoUrl: 'https://forge.softwareheritage.org/', ], + {%- endif %} ]) } } stage('Build and upload PyPI package') { when { + {%- if gitlab_project %} + expression { "${env.gitlabSourceBranch}" ==~ /refs\/tags\/v\d+(.\d+)+/ } + expression { jobExists('/{{name}}/gitlab-pypi-upload') } + {%- else %} expression { params.GIT_TAG ==~ /v\d+(.\d+)+/ } expression { jobExists('/{{name}}/pypi-upload') } + {%- endif %} + } steps { build( + {%- if gitlab_project %} + job: '/{{name}}/gitlab-pypi-upload', + {%- else %} job: '/{{name}}/pypi-upload', + {%- endif %} parameters: [ + {%- if gitlab_project %} + string(name: 'GIT_TAG', value: "${env.gitlabSourceBranch}"), + {%- else %} string(name: 'GIT_TAG', value: params.GIT_TAG), + {%- endif %} string(name: 'PYPI_HOST', value: '{{incoming_tag_auto_pypi_host}}'), ], ) } } + {%- if not gitlab_project %} stage('Update Debian packaging for new release') { when { expression { params.GIT_TAG ==~ /v\d+(.\d+)+/ } expression { jobExists('/debian/packages/{{name}}/update-for-release') } } steps { build( job: '/debian/packages/{{name}}/update-for-release', parameters: [ string(name: 'GIT_TAG', value: params.GIT_TAG), ], wait: false, ) } } stage('Build Debian package') { when { expression { params.GIT_TAG ==~ /debian\/.*/ } expression { !(params.GIT_TAG ==~ /debian\/upstream\/.*/) } expression { jobExists('/debian/packages/{{name}}/gbp-buildpackage') } } steps { build( job: '/debian/packages/{{name}}/gbp-buildpackage', parameters: [ string(name: 'GIT_REVISION', value: params.GIT_TAG), booleanParam(name: 'DO_UPLOAD', value: true), ], wait: false, ) } } + {%- endif %} } } diff --git a/jobs/templates/incoming-tag.yaml b/jobs/templates/incoming-tag.yaml index 1fe1950..97dd831 100644 --- a/jobs/templates/incoming-tag.yaml +++ b/jobs/templates/incoming-tag.yaml @@ -1,25 +1,38 @@ - job-template: &incoming_tag name: "{name}/incoming-tag" display-name: Incoming tag project-type: pipeline - auth-token: 'ph4br1cat0r' + auth-token: "ph4br1cat0r" incoming_tag_auto_pypi_host: pypi.org sandbox: true + gitlab_project: false properties: - build-discarder: num-to-keep: 20 parameters: - git-parameter: name: GIT_TAG description: git tag to process type: PT_TAG sortMode: DESCENDING_SMART selectedValue: TOP - dsl: - !include-jinja2: incoming-tag.groovy.j2 + dsl: !include-jinja2: incoming-tag.groovy.j2 - job-template: name: "debian/deps/{name}/incoming-tag" - dsl: - !include-jinja2: dependency-incoming-tag.groovy.j2 + dsl: !include-jinja2: dependency-incoming-tag.groovy.j2 + <<: *incoming_tag + +- job-template: + name: "{name}/gitlab-incoming-tag" + display-name: Incoming tag (GitLab) + gitlab_project: true + parameters: + triggers: + - gitlab: + trigger-push: true + trigger-merge-request: false + add-ci-message: true + # secret jenkins token is generated when executing tox + secret-token: !include-raw: jenkins-token <<: *incoming_tag diff --git a/jobs/templates/swh-pypi.groovy.j2 b/jobs/templates/swh-pypi.groovy.j2 index 859becf..853c21c 100644 --- a/jobs/templates/swh-pypi.groovy.j2 +++ b/jobs/templates/swh-pypi.groovy.j2 @@ -1,111 +1,129 @@ def module_name = '{{repo_name}}'.replace('-', '.') def PYPI_UPLOAD_HOST switch (params.PYPI_HOST) { case 'pypi.org': PYPI_UPLOAD_HOST = 'upload.pypi.org' break default: PYPI_UPLOAD_HOST = params.PYPI_HOST break } def BDIST_WHEEL = 'bdist_wheel' if (!params.INCLUDE_BDIST) { BDIST_WHEEL = '' } pipeline { {% filter indent(width=2) %} {%- include 'includes/agent-docker.groovy.j2' -%} {% endfilter %} stages { stage('Run tests') { when { expression { return !params.SKIP_TESTS } beforeAgent true } agent none steps { build( + {%- if gitlab_project %} + job: '/{{name}}/gitlab-tests', + {%- else %} job: '/{{name}}/tests', + {%- endif %} parameters: [ string(name: 'REVISION', value: params.GIT_TAG), ], propagate: !params.IGNORE_TESTS, ) } } stage('Checkout') { steps { - checkout([$class: 'GitSCM', - branches: [[name: params.GIT_TAG]], - doGenerateSubmoduleConfigurations: false, - extensions: [], - gitTool: 'Default', - submoduleCfg: [], - userRemoteConfigs: [[url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git']] + checkout([ + $class: 'GitSCM', + {%- if gitlab_project %} + userRemoteConfigs: [[ + name:'origin', url: '{{gitlab_url}}/{{gitlab_project_name}}.git', + refspec: '+refs/tags/*:refs/remotes/origin/tags*' + ]], + branches: [[ + name: "${params.GIT_TAG}" + ]], + extensions: [[$class: 'CloneOption', honorRefspec: true]], + {%- else %} + userRemoteConfigs: [[url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git']], + branches: [[name: params.GIT_TAG]], + {%- endif %} + doGenerateSubmoduleConfigurations: false, + extensions: [], + gitTool: 'Default', + submoduleCfg: [], ]) sh '''rm -rf dist/''' } } stage('Build') { steps { withCredentials([ string(credentialsId: 'sentry-auth-token', variable: 'SENTRY_AUTH_TOKEN')]) { sh """ # Build javascript assets if [ -f yarn.lock ]; then yarn install --frozen-lockfile yarn build fi # Build java assets if [ -d java ]; then for pom in \$( find java/ -name pom.xml ) ; do mvn -f \$pom compile assembly:single done fi python3 setup.py sdist ${BDIST_WHEEL} """ archiveArtifacts allowEmptyArchive: true, artifacts: 'dist/*', fingerprint: true } } } + {%- if not gitlab_project %} stage('Publish') { when { anyOf { expression { return params.FORCE_UPLOAD } expression { LASTV=sh(returnStdout: true, script:"curl -s https://${params.PYPI_HOST}/pypi/${module_name}/json | jq -r .info.version || true").trim() return 'v'+LASTV != params.GIT_TAG } } } steps { withCredentials([ usernamePassword(credentialsId: PYPI_UPLOAD_HOST, usernameVariable: 'TWINE_USERNAME', passwordVariable: 'TWINE_PASSWORD')]) { sh "python3 -m twine upload --verbose --repository-url https://${PYPI_UPLOAD_HOST}/legacy/ dist/*" } } } + {%- endif %} } post { cleanup { cleanWs() } } } diff --git a/jobs/templates/swh-pypi.yaml b/jobs/templates/swh-pypi.yaml index 9d4cc07..27123f3 100644 --- a/jobs/templates/swh-pypi.yaml +++ b/jobs/templates/swh-pypi.yaml @@ -1,52 +1,58 @@ -- job-template: +- job-template: &pypi_upload name: "{name}/pypi-upload" display-name: "Upload to PyPI" docker_image: tox include_bdist: true project-type: pipeline sandbox: true + gitlab_project: false properties: - build-discarder: artifact-num-to-keep: 10 - copyartifact: - projects: '*' + projects: "*" parameters: - git-parameter: name: GIT_TAG description: git tag to process type: PT_TAG sortMode: DESCENDING_SMART selectedValue: TOP - choice: name: PYPI_HOST description: | PYPI registry to publish to when built revision is a tag. There must exists a usernamePassword credential object with that name. The pypi JSON API endpoint is built as https://$PYPI_HOST/project//json choices: - test.pypi.org - pypi.org default: test.pypi.org - bool: name: FORCE_UPLOAD default: false description: | Force uploading python packages on the chosen PYPI registry, even if the package already exists. - bool: name: SKIP_TESTS default: false description: | Do not run tests on the repository. - bool: name: IGNORE_TESTS default: false description: | Proceed even if the tests are failing on the repository. - bool: name: INCLUDE_BDIST - default: '{include_bdist}' + default: "{include_bdist}" description: | Include a binary distribution in the PyPI upload. - dsl: - !include-jinja2: swh-pypi.groovy.j2 + dsl: !include-jinja2: swh-pypi.groovy.j2 + +- job-template: + name: "{name}/gitlab-pypi-upload" + display-name: "Upload to PyPI (GitLab)" + gitlab_project: true + <<: *pypi_upload diff --git a/jobs/tools/setup-gitlab-webhooks.groovy.j2 b/jobs/tools/setup-gitlab-webhooks.groovy.j2 index 900fc30..5976ce5 100644 --- a/jobs/tools/setup-gitlab-webhooks.groovy.j2 +++ b/jobs/tools/setup-gitlab-webhooks.groovy.j2 @@ -1,84 +1,86 @@ pipeline { agent any environment { GITLAB_TOKEN = credentials("jenkins-gitlab-token") } stages { stage('Checkout Repository') { steps { checkout([ $class: 'GitSCM', branches: [[name: 'master']], userRemoteConfigs: [[ url: "http://forge.softwareheritage.org/source/swh-jenkins-jobs.git", ]], ]) } } stage('Setup gitlab integration') { steps { script { setupGitlabWebhook("swh/infra/ci-cd/swh-jenkins-jobs", "jenkins-tools/swh-jenkins-jobs-builder", true, true, false) projects = readYaml(file: 'jobs/swh-packages.yaml') for (project in projects) { if (project.containsKey("project")) { def jenkinsFolder = project.get('project').get('name') def repoName= project.get('project').get('repo_name') def gitlabProjectName = "swh/devel/${repoName}" setupGitlabWebhook(gitlabProjectName, "${jenkinsFolder}/gitlab-tests") + setupGitlabWebhook(gitlabProjectName, "${jenkinsFolder}/gitlab-incoming-tag", + false, false, true) } } } } } } } void setupGitlabWebhook(gitlabProjectName, jenkinsProjectName, pushEvents = true, mergeRequestEvents = true, tagPushEvents = false) { def webhookUrl = "${jenkins_url}/project/${jenkinsProjectName}" def gitlabProjectEncoded = java.net.URLEncoder.encode(gitlabProjectName, "UTF-8") def payload = """ { "id": "${gitlabProjectEncoded}", "url": "${webhookUrl}", "push_events": "${pushEvents}", "merge_requests_events": "${mergeRequestEvents}", "tag_push_events": "${tagPushEvents}", "token": "{{jenkins_token}}" } """ def url = "${gitlab_url}/api/v4/projects/${gitlabProjectEncoded}/hooks" // get current webhooks of the gitlab project def response = httpRequest httpMode: 'GET', url: url, contentType: 'APPLICATION_JSON', customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]] def hooks = readJSON text: response.content for (hook in hooks) { if (hook.url == webhookUrl) { // update previously set webhook for jenkins job httpRequest httpMode: 'PUT', url: "${url}/${hook.id}", contentType: 'APPLICATION_JSON', requestBody: payload, customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]] return } } // add webhook for jenkins job httpRequest httpMode: 'POST', url: url, contentType: 'APPLICATION_JSON', requestBody: payload, customHeaders: [[name: 'Authorization', value: "Bearer $GITLAB_TOKEN"]] }