diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -1747,9 +1747,10 @@ swh::deploy::journal::backfill::user: swhstorage swh::deploy::journal::backfill::group: swhstorage +swh::deploy::storage::db::config::read-only: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=guest dbname=%{hiera('swh::deploy::storage::db::dbname')} password=guest" swh::deploy::storage::config::local-read-only: cls: local - db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=guest dbname=%{hiera('swh::deploy::storage::db::dbname')} password=guest" + db: "%{alias('swh::deploy::storage::db::config::read-only')}" # objstorage is required but it's not used in that context objstorage: cls: memory @@ -1888,6 +1889,38 @@ swh::deploy::journal::prefix: swh.journal.objects +swh::deploy::scrubber::sentry_swh_package: swh.scrubber +swh::deploy::scrubber::sentry_environment: "%{alias('swh::deploy::environment')}" +swh::deploy::scrubber::sentry_dsn: "https://%{lookup('swh::deploy::scrubber::sentry_token')}@sentry.softwareheritage.org/23" + +swh::deploy::scrubber::checker::postgres::conf_directory: "%{hiera('swh::conf_directory')}/scrubber" +swh::deploy::scrubber::checker::postgres::conf_file: "%{hiera('swh::deploy::scrubber::checker::postgres::conf_directory')}/postgres.yml" +swh::deploy::scrubber::checker::postgres::user: swhworker +swh::deploy::scrubber::checker::postgres::group: swhdev +swh::deploy::scrubber::checker::postgres::object_types: + # - content + # - directory + # - revision + - release + +swh::deploy::scrubber::checker::postgres::ranges: + - 00000000000000000000:44444444444444444444 + - 44444444444444444444:88888888888888888888 + - 88888888888888888888:bbbbbbbbbbbbbbbbbbbb + - bbbbbbbbbbbbbbbbbbbb:ffffffffffffffffffff + +# password entry in private-data +swh::deploy::scrubber::db::config: "host=%{hiera('swh::deploy::scrubber::db::host')} port=%{hiera('swh::deploy::db::pgbouncer::port')} dbname=%{hiera('swh::deploy::scrubber::db::dbname')} user=%{hiera('swh::deploy::scrubber::db::user')} password=%{hiera('swh::deploy::scrubber::db::password')}" + +swh::deploy::scrubber::checker::postgres::config: + scrubber_db: + cls: local + db: "%{alias('swh::deploy::scrubber::db::config')}" + storage: + cls: postgresql + db: "%{alias('swh::deploy::storage::db::config::read-only')}" + objstorage: "%{alias('swh::remote_service::objstorage::config')}" + swh::deploy::journal_simple_checker_producer::conf_file: "%{hiera('swh::deploy::journal::conf_directory')}/checker.yml" swh::deploy::journal_simple_checker_producer::user: swhstorage swh::deploy::journal_simple_checker_producer::group: swhstorage @@ -3769,3 +3802,4 @@ sonatype: http://oss.sonatype.org/content/repositories/releases/ jboss: https://repository.jboss.org/maven2/ atlassian-public: https://maven.atlassian.com/public/ + diff --git a/data/subnets/vagrant.yaml b/data/subnets/vagrant.yaml --- a/data/subnets/vagrant.yaml +++ b/data/subnets/vagrant.yaml @@ -239,6 +239,8 @@ host: worker3.internal.staging.swh.network 10.168.130.110: host: objstorage0.internal.staging.swh.network + 10.168.130.120: + host: scrubber0.internal.staging.swh.network 10.168.130.160: host: mirror-test.internal.staging.swh.network 10.168.200.22: diff --git a/manifests/site.pp b/manifests/site.pp --- a/manifests/site.pp +++ b/manifests/site.pp @@ -183,6 +183,10 @@ include role::swh_counters_with_journal_client } +node /^scrubber\d+\.internal\.staging\.swh\.network/ { + include role::swh_scrubber_checkers +} + node 'webapp.internal.staging.swh.network' { include role::swh_webapp } diff --git a/site-modules/profile/files/swh/deploy/scrubber/swh-scrubber.slice b/site-modules/profile/files/swh/deploy/scrubber/swh-scrubber.slice new file mode 100644 --- /dev/null +++ b/site-modules/profile/files/swh/deploy/scrubber/swh-scrubber.slice @@ -0,0 +1,5 @@ +[Unit] +Description=SWH Scrubber services + +[Slice] +MemoryMax=10% diff --git a/site-modules/profile/manifests/swh/deploy/scrubber/checker/postgres.pp b/site-modules/profile/manifests/swh/deploy/scrubber/checker/postgres.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/scrubber/checker/postgres.pp @@ -0,0 +1,90 @@ +# Deployment of the swh.scrubber's checker postgres service + +class profile::swh::deploy::scrubber::checker::postgres { + $sentry_dsn = lookup("swh::deploy::scrubber::sentry_dsn") + $sentry_environment = lookup("swh::deploy::scrubber::sentry_environment") + $sentry_swh_package = lookup("swh::deploy::scrubber::sentry_swh_package") + + $config_dir = lookup('swh::deploy::scrubber::checker::postgres::conf_directory') + $config_file = lookup('swh::deploy::scrubber::checker::postgres::conf_file') + $config_dict = lookup('swh::deploy::scrubber::checker::postgres::config') + $user = lookup('swh::deploy::scrubber::checker::postgres::user') + $group = lookup('swh::deploy::scrubber::checker::postgres::group') + + $object_types = lookup('swh::deploy::scrubber::checker::postgres::object_types') + $ranges = lookup('swh::deploy::scrubber::checker::postgres::ranges') + + $packages = ['python3-swh.scrubber'] + ensure_packages($packages) + + file {$config_dir: + ensure => 'directory', + owner => $user, + group => $group, + } + + file {$config_file: + ensure => present, + owner => $user, + group => $group, + mode => '0640', + content => inline_template("<%= @config_dict.to_yaml %>\n"), + require => File[$config_dir] + } + + $systemd_slice_name = "swh-scrubber.slice" + ::systemd::unit_file {$systemd_slice_name: + ensure => 'present', + source => "puppet:///modules/profile/swh/deploy/scrubber/${systemd_slice_name}", + } + + $template_name = 'swh-scrubber-checker-postgres' + $template_unit_name = "${template_name}@.service" + # Template uses: + # - $user + # - $group + # - $sentry_dsn + # - $sentry_environment + # - $sentry_swh_package + # - $config_file + ::systemd::unit_file {$template_unit_name: + ensure => present, + content => template("profile/swh/deploy/scrubber/${template_unit_name}.erb"), + enable => false, + require => [ + File[$config_file], + ::Systemd::Unit_file[$systemd_slice_name], + Package[$packages], + ] + } + + $object_types.each | $object_type | { + $ranges.each | $range_index, $range | { + $ranges_list = $range.split(':') + $start_object = $ranges_list[0] + $end_object = $ranges_list[1] + $service_name = "${template_name}@${object_type}-${range_index}.service" + + $parameters_conf_name = "${service_name}.d/parameters.conf" + # Template uses: + # - $object_type + # - $start_object + # - $end_object + ::systemd::dropin_file {$parameters_conf_name: + ensure => present, + unit => $service_name, + filename => 'parameters.conf', + content => template("profile/swh/deploy/scrubber/parameters.conf.erb"), + } + + service {$service_name: + ensure => running, + enable => false, + require => [ + ::Systemd::Unit_file[$template_unit_name], + ::Systemd::Dropin_File[$parameters_conf_name], + ], + } + } + } +} diff --git a/site-modules/profile/templates/swh/deploy/scrubber/parameters.conf.erb b/site-modules/profile/templates/swh/deploy/scrubber/parameters.conf.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/swh/deploy/scrubber/parameters.conf.erb @@ -0,0 +1,5 @@ +# Managed by puppet class profile::swh::deploy::scrubber::checker::postgres +# Modifications will be overwritten + +[Service] +Environment=SWH_SCRUBBER_CLI_EXTRA_ARGS="--object-type <%= @object_type %> --start-object <%= @start_object %> --end-object <%= @end_object %>" diff --git a/site-modules/profile/templates/swh/deploy/scrubber/swh-scrubber-checker-postgres@.service.erb b/site-modules/profile/templates/swh/deploy/scrubber/swh-scrubber-checker-postgres@.service.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/swh/deploy/scrubber/swh-scrubber-checker-postgres@.service.erb @@ -0,0 +1,26 @@ +# Scrubber checker postgres template unit file +# Managed by puppet class profile::swh::deploy::scrubber::checker::postgres +# Changes will be overwritten + +[Unit] +Description=Software Heritage Scrubber Checker Postgres %i +After=network.target + +[Service] +<%- if !@sentry_dsn.nil? and !@sentry_dsn.empty? -%> +Environment=SWH_SENTRY_DSN=<%= @sentry_dsn %> +<%- end -%> +<%- if !@sentry_environment.nil? and !@sentry_environment.empty? -%> +Environment=SWH_SENTRY_ENVIRONMENT=<%= @sentry_environment %> +<%- end -%> +<%- if !@sentry_swh_package.nil? and !@sentry_swh_package.empty? -%> +Environment=SWH_MAIN_PACKAGE=<%= @sentry_swh_package %> +<%- end -%> +Environment=SWH_SCRUBBER_CLI_EXTRA_ARGS="--help" +User=<%= @user %> +Group=<%= @group %> +Type=simple +ExecStart=/usr/bin/swh scrubber --config-file <%= @config_file %> check storage ${SWH_SCRUBBER_CLI_EXTRA_ARGS} + +[Install] +WantedBy=multi-user.target diff --git a/site-modules/role/manifests/swh_scrubber_checkers.pp b/site-modules/role/manifests/swh_scrubber_checkers.pp new file mode 100644 --- /dev/null +++ b/site-modules/role/manifests/swh_scrubber_checkers.pp @@ -0,0 +1,4 @@ +class role::swh_scrubber_checkers inherits role::swh_base { + # checker(s) + include profile::swh::deploy::scrubber::checker::postgres +} diff --git a/site-modules/role/manifests/swh_storage_with_journal.pp b/site-modules/role/manifests/swh_storage_with_journal.pp --- a/site-modules/role/manifests/swh_storage_with_journal.pp +++ b/site-modules/role/manifests/swh_storage_with_journal.pp @@ -2,7 +2,7 @@ include profile::postgresql::client include profile::swh::deploy::journal::backfill - # journal + # journal include profile::zookeeper include profile::kafka::broker }