diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -1104,6 +1104,10 @@ type: CNAME record: azure-billing.internal.admin.swh.network data: money.internal.admin.swh.network. + maven-index-exporter/CNAME: + type: CNAME + record: maven-exporter.internal.staging.swh.network + data: maven-exporter0.internal.staging.swh.network. # Non-puppet azure hosts pgmirror0.euwest.azure/A: record: pgmirror0.euwest.azure.internal.softwareheritage.org @@ -3734,3 +3738,10 @@ azure_billing::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" azure_billing::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" azure_billing::vhost::hsts_header: "%{hiera('apache::hsts_header')}" + +maven_index_exporter::image::name: softwareheritage/maven-index-exporter +maven_index_exporter::image::version: v0.2.0 + +maven_index_exporter::repositories: + maven-central: https://repo1.maven.org/maven2/ + clojars: http://clojars.org/repo/ diff --git a/data/subnets/vagrant.yaml b/data/subnets/vagrant.yaml --- a/data/subnets/vagrant.yaml +++ b/data/subnets/vagrant.yaml @@ -219,6 +219,8 @@ host: scheduler0.internal.staging.swh.network 10.168.130.60: host: vault.internal.staging.swh.network + 10.168.130.70: + host: maven-exporter0.internal.staging.swh.network 10.168.130.80: host: search-esnode0.internal.staging.swh.network 10.168.130.90: diff --git a/manifests/site.pp b/manifests/site.pp --- a/manifests/site.pp +++ b/manifests/site.pp @@ -223,6 +223,10 @@ include role::swh_azure_billing_report } +node 'maven-exporter0.internal.staging.swh.network' { + include role::swh_maven_index_exporter +} + node default { include role::swh_base } diff --git a/site-modules/profile/files/maven_index_exporter/maven_index_exporter.slice b/site-modules/profile/files/maven_index_exporter/maven_index_exporter.slice new file mode 100644 --- /dev/null +++ b/site-modules/profile/files/maven_index_exporter/maven_index_exporter.slice @@ -0,0 +1,5 @@ +[Unit] +Description=Maven index exporters + +[Slice] +MemoryMax=75% diff --git a/site-modules/profile/manifests/maven_index_exporter.pp b/site-modules/profile/manifests/maven_index_exporter.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/maven_index_exporter.pp @@ -0,0 +1,97 @@ +# Deployment for maven index exporter +class profile::maven_index_exporter { + $user = 'root' + $group = 'root' + + $publish_path = '/var/www/maven_index_exporter' + + $base_dir = "/srv/softwareheritage/maven-index-exporter/" + $docker_image = lookup('maven_index_exporter::image::name') + $docker_image_version = lookup('maven_index_exporter::image::version') + $mvn_repositories = lookup('maven_index_exporter::repositories') + + # Create the base directory + file { $base_dir: + ensure => directory, + recurse => true, + owner => $user, + group => 'root', + } + $template_name = 'maven_index_exporter' + $template_path = "profile/${template_name}" + + $script_name = "run_maven_index_exporter.sh" + # Template use: + # - $base_dir + # - $publish_path + # - $docker_image + # - $docker_image_version + file { "/usr/local/bin/${script_name}": + ensure => present, + content => template("${template_path}/${script_name}.erb"), + owner => $user, + group => 'root', + mode => '0744' + } + + # Install the publish path + file { $publish_path: + ensure => directory, + owner => $user, + group => 'root', + recurse => true, + } + + # Service declaration + + $systemd_basename = "${template_name}@" + $systemd_timer = "${systemd_basename}.timer" + $systemd_service = "${systemd_basename}.service" + + # Declare: maven_index_exporter@.service and maven_index_exporter@.timer + + # Timer and service declaration + # timer template uses: + # - $template_name + # Service template uses: + # - $user + # - $group + ::systemd::timer { $systemd_timer: + timer_content => template("${template_path}/${systemd_timer}.erb"), + service_content => template("${template_path}/${systemd_service}.erb"), + enable => true, + } + + $systemd_slice_name = "${template_name}.slice" + ::systemd::unit_file {$systemd_slice_name: + ensure => 'present', + source => "puppet:///modules/profile/${template_name}/${systemd_slice_name}", + } + + # Iterate over the maven repositories to extract index out of + # and install configuration + $mvn_repositories.each | $mvn_repo_name, $mvn_repo_url | { + # systemd service declaration per maven repository + $service_basename = "${template_name}@${mvn_repo_name}" + $service_name = "${service_basename}.service" + # Template use: + # - $mvn_repo_url + ::systemd::dropin_file {"${service_name}.d/parameters.conf": + ensure => present, + unit => $service_name, + filename => 'parameters.conf', + content => template("${template_path}/parameters.conf.erb"), + } + + ::systemd::unit_file {$service_name: + ensure => present, + enable => true, + } + $service_timer = "${service_basename}.timer" + ::systemd::timer {$service_timer: + ensure => present, + enable => true, + active => true, + } + } +} diff --git a/site-modules/profile/templates/maven_index_exporter/maven_index_exporter@.service.erb b/site-modules/profile/templates/maven_index_exporter/maven_index_exporter@.service.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/maven_index_exporter/maven_index_exporter@.service.erb @@ -0,0 +1,20 @@ +# Maven index exporter Template unit file +# Managed by puppet class profile::maven_index_exporter +# Changes will be overwritten + +[Unit] +Description=Software Heritage Maven Index Exporter %i +StartLimitIntervalSec=5min +StartLimitBurst=2 +After=network.target + +[Service] +Type=simple +User=<%= @user %> +Group=<%= @group %> +ExecStart=/usr/local/bin/run_maven_index_exporter.sh %i +Restart=on-failure +RestartSec=1s + +[Install] +WantedBy=multi-user.target diff --git a/site-modules/profile/templates/maven_index_exporter/maven_index_exporter@.timer.erb b/site-modules/profile/templates/maven_index_exporter/maven_index_exporter@.timer.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/maven_index_exporter/maven_index_exporter@.timer.erb @@ -0,0 +1,10 @@ +[Install] +WantedBy=timers.target + +[Timer] +OnCalendar=daily +Persistent=true +RandomizedDelaySec=24h + +[Unit] +Description=Run '<%= @template_name %>@.service' daily diff --git a/site-modules/profile/templates/maven_index_exporter/parameters.conf.erb b/site-modules/profile/templates/maven_index_exporter/parameters.conf.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/maven_index_exporter/parameters.conf.erb @@ -0,0 +1,5 @@ +# Managed by puppet - modifications will be overwritten +# In defined class profile::maven_index_exporter + +[Service] +Environment=MAVEN_REPO_URL=<%= @mvn_repo_url %> diff --git a/site-modules/profile/templates/maven_index_exporter/run_maven_index_exporter.sh.erb b/site-modules/profile/templates/maven_index_exporter/run_maven_index_exporter.sh.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/maven_index_exporter/run_maven_index_exporter.sh.erb @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Managed by puppet - modifications will be overwritten +# In defined class profile::maven_index_exporter + +# Script in charge of running the maven index exporter for a specific instance +# The most importand part is the result stored in $PUBLISH_DIR/export-${maven-repo}.fld + +set -ex + +# Unique maven repository instance name (e.g. maven-central) +MAVEN_REPO=$1 + +if [ -z "${MAVEN_REPO_URL}" ]; then + echo MAVEN_REPO_URL must be set. + exit 1 +fi + +BASE_DIR=<%= @base_dir %> +WORK_DIR=$BASE_DIR/$MAVEN_REPO/work +PUBLISH_DIR=<%= @publish_path %> + +# Create folder +mkdir -p $WORK_DIR $PUBLISH_DIR + +DOCKER_IMAGE=<%= @docker_image %> +DOCKER_IMAGE_VERSION=<%= @docker_image_version %> + +# Compute and extract the /publish/export.fld +docker run -v $WORK_DIR:/work \ + -v $PUBLISH_DIR:/publish \ + -e MVN_IDX_EXPORTER_BASE_URL=$MAVEN_REPO_URL \ + "${DOCKER_IMAGE}:${DOCKER_IMAGE_VERSION}" + +# Rename and install the export.fld (specific to the maven index base url scrapped) +mv $PUBLISH_DIR/export.fld $PUBLISH_DIR/export-${MAVEN_REPO}.fld diff --git a/site-modules/role/manifests/swh_maven_index_exporter.pp b/site-modules/role/manifests/swh_maven_index_exporter.pp new file mode 100644 --- /dev/null +++ b/site-modules/role/manifests/swh_maven_index_exporter.pp @@ -0,0 +1,5 @@ +# SWH maven index exporter service +class role::swh_maven_index_exporter inherits role::swh_base { + include profile::docker + include profile::maven_index_exporter +}