Page MenuHomeSoftware Heritage
Files F9312623

D8181.diff
No OneTemporary
Actions

D8181.diff
View Options

diff --git a/data/common/common.yaml b/data/common/common.yaml
--- a/data/common/common.yaml
+++ b/data/common/common.yaml
@@ -1750,6 +1750,7 @@
swh::deploy::journal::backfill::group: swhstorage
swh::deploy::storage::db::config::read-only: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=guest dbname=%{hiera('swh::deploy::storage::db::dbname')} password=guest"
+swh::deploy::storage::db::secondary::config::read-only: "host=somerset.internal.softwareheritage.org port=%{hiera('swh::deploy::storage::db::port')} user=guest dbname=%{hiera('swh::deploy::storage::db::dbname')} password=guest"
swh::deploy::storage::config::local-read-only:
cls: postgresql
db: "%{alias('swh::deploy::storage::db::config::read-only')}"
@@ -1901,7 +1902,6 @@
swh::deploy::scrubber::sentry_dsn: "https://%{lookup('swh::deploy::scrubber::sentry_token')}@sentry.softwareheritage.org/23"
swh::deploy::scrubber::checker::postgres::conf_directory: "%{hiera('swh::conf_directory')}/scrubber"
-swh::deploy::scrubber::checker::postgres::conf_file: "%{hiera('swh::deploy::scrubber::checker::postgres::conf_directory')}/postgres.yml"
swh::deploy::scrubber::checker::postgres::user: swhworker
swh::deploy::scrubber::checker::postgres::group: swhdev
swh::deploy::scrubber::checker::postgres::object_types:
@@ -1911,10 +1911,22 @@
- snapshot
swh::deploy::scrubber::checker::postgres::ranges:
- - 00000000000000000000:44444444444444444444
- - 44444444444444444444:88888888888888888888
- - 88888888888888888888:bbbbbbbbbbbbbbbbbbbb
- - bbbbbbbbbbbbbbbbbbbb:ffffffffffffffffffff
+ - 0000000000000000000000000000000000000000:3fffffffffffffffffffffffffffffffffffffff
+ - 4000000000000000000000000000000000000000:7fffffffffffffffffffffffffffffffffffffff
+ - 8000000000000000000000000000000000000000:afffffffffffffffffffffffffffffffffffffff
+ - b000000000000000000000000000000000000000:ffffffffffffffffffffffffffffffffffffffff
+
+swh::deploy::scrubber::checker::postgres::config_per_db:
+ primary:
+ config:
+ scrubber_db:
+ cls: postgresql
+ db: "%{alias('swh::deploy::scrubber::db::config')}"
+ storage:
+ cls: postgresql
+ db: "%{alias('swh::deploy::storage::db::config::read-only')}"
+ objstorage:
+ cls: noop
# password entry in private-data
swh::deploy::scrubber::db::config: "host=%{hiera('swh::deploy::scrubber::db::host')} port=%{hiera('swh::deploy::db::pgbouncer::port')} dbname=%{hiera('swh::deploy::scrubber::db::dbname')} user=%{hiera('swh::deploy::scrubber::db::user')} password=%{hiera('swh::deploy::scrubber::db::password')}"
diff --git a/data/deployments/production/sesi_rocquencourt.yaml b/data/deployments/production/sesi_rocquencourt.yaml
--- a/data/deployments/production/sesi_rocquencourt.yaml
+++ b/data/deployments/production/sesi_rocquencourt.yaml
@@ -31,3 +31,25 @@
- nodev
- relatime
- rw
+
+swh::deploy::scrubber::checker::postgres::config_per_db:
+ primary:
+ config:
+ scrubber_db:
+ cls: postgresql
+ db: "%{alias('swh::deploy::scrubber::db::config')}"
+ storage:
+ cls: postgresql
+ db: "%{alias('swh::deploy::storage::db::config::read-only')}"
+ objstorage:
+ cls: noop
+ secondary:
+ config:
+ scrubber_db:
+ cls: postgresql
+ db: "%{alias('swh::deploy::scrubber::db::config')}"
+ storage:
+ cls: postgresql
+ db: "%{alias('swh::deploy::storage::db::secondary::config::read-only')}"
+ objstorage:
+ cls: noop
diff --git a/site-modules/profile/manifests/swh/deploy/scrubber/checker/postgres.pp b/site-modules/profile/manifests/swh/deploy/scrubber/checker/postgres.pp
--- a/site-modules/profile/manifests/swh/deploy/scrubber/checker/postgres.pp
+++ b/site-modules/profile/manifests/swh/deploy/scrubber/checker/postgres.pp
@@ -6,8 +6,6 @@
$sentry_swh_package = lookup("swh::deploy::scrubber::sentry_swh_package")
$config_dir = lookup('swh::deploy::scrubber::checker::postgres::conf_directory')
- $config_file = lookup('swh::deploy::scrubber::checker::postgres::conf_file')
- $config_dict = lookup('swh::deploy::scrubber::checker::postgres::config')
$user = lookup('swh::deploy::scrubber::checker::postgres::user')
$group = lookup('swh::deploy::scrubber::checker::postgres::group')
@@ -21,15 +19,9 @@
ensure => 'directory',
owner => $user,
group => $group,
- }
-
- file {$config_file:
- ensure => present,
- owner => $user,
- group => $group,
- mode => '0640',
- content => inline_template("<%= @config_dict.to_yaml %>\n"),
- require => File[$config_dir]
+ purge => true,
+ force => true,
+ recurse => true,
}
$systemd_slice_name = "swh-scrubber.slice"
@@ -46,7 +38,6 @@
# - $sentry_dsn
# - $sentry_environment
# - $sentry_swh_package
- # - $config_file
::systemd::unit_file {$template_unit_name:
ensure => present,
content => template("profile/swh/deploy/scrubber/${template_unit_name}.erb"),
@@ -58,32 +49,68 @@
]
}
+ $config_per_dbs_to_scrub = lookup('swh::deploy::scrubber::checker::postgres::config_per_db')
+
+ # As many services as there are dbs to scrub
+ $config_per_dbs_to_scrub.each | $db_name, $cfg | {
+ $config_file = "${config_dir}/${db_name}.yaml"
+ $config_dict = $cfg['config']
+ file {$config_file:
+ ensure => present,
+ owner => $user,
+ group => $group,
+ mode => '0640',
+ content => inline_yaml($config_dict),
+ require => File[$config_dir]
+ }
+
+ $object_types.each | $object_type | {
+ $ranges.each | $range_index, $range | {
+ $ranges_list = $range.split(':')
+ $start_object = $ranges_list[0]
+ $end_object = $ranges_list[1]
+ $service_name = "${template_name}@${db_name}-${object_type}-${range_index}.service"
+
+ $parameters_conf_name = "${service_name}.d/parameters.conf"
+ # Template uses:
+ # - $object_type
+ # - $start_object
+ # - $end_object
+ # - $config_file
+ ::systemd::dropin_file {$parameters_conf_name:
+ ensure => present,
+ unit => $service_name,
+ filename => 'parameters.conf',
+ content => template("profile/swh/deploy/scrubber/parameters.conf.erb"),
+ }
+
+ service {$service_name:
+ ensure => running,
+ enable => true,
+ require => [
+ ::Systemd::Unit_file[$template_unit_name],
+ ::Systemd::Dropin_File[$parameters_conf_name],
+ ],
+ }
+ }
+ }
+ }
+
+ # clean up old resources
$object_types.each | $object_type | {
$ranges.each | $range_index, $range | {
- $ranges_list = $range.split(':')
- $start_object = $ranges_list[0]
- $end_object = $ranges_list[1]
- $service_name = "${template_name}@${object_type}-${range_index}.service"
+ $old_svc_name = "${template_name}@${object_type}-${range_index}.service"
+ $old_params_confname = "${old_svc_name}.d/parameters.conf"
- $parameters_conf_name = "${service_name}.d/parameters.conf"
- # Template uses:
- # - $object_type
- # - $start_object
- # - $end_object
- ::systemd::dropin_file {$parameters_conf_name:
- ensure => present,
- unit => $service_name,
+ ::systemd::dropin_file {$old_params_confname:
+ ensure => absent,
+ unit => $old_svc_name,
filename => 'parameters.conf',
- content => template("profile/swh/deploy/scrubber/parameters.conf.erb"),
}
- service {$service_name:
- ensure => running,
+ service {$old_svc_name:
+ ensure => stopped,
enable => false,
- require => [
- ::Systemd::Unit_file[$template_unit_name],
- ::Systemd::Dropin_File[$parameters_conf_name],
- ],
}
}
}
diff --git a/site-modules/profile/templates/swh/deploy/scrubber/parameters.conf.erb b/site-modules/profile/templates/swh/deploy/scrubber/parameters.conf.erb
--- a/site-modules/profile/templates/swh/deploy/scrubber/parameters.conf.erb
+++ b/site-modules/profile/templates/swh/deploy/scrubber/parameters.conf.erb
@@ -3,3 +3,4 @@
[Service]
Environment=SWH_SCRUBBER_CLI_EXTRA_ARGS="--object-type <%= @object_type %> --start-object <%= @start_object %> --end-object <%= @end_object %>"
+SWH_CONFIG_FILENAME=<%= @config_file %>
diff --git a/site-modules/profile/templates/swh/deploy/scrubber/swh-scrubber-checker-postgres@.service.erb b/site-modules/profile/templates/swh/deploy/scrubber/swh-scrubber-checker-postgres@.service.erb
--- a/site-modules/profile/templates/swh/deploy/scrubber/swh-scrubber-checker-postgres@.service.erb
+++ b/site-modules/profile/templates/swh/deploy/scrubber/swh-scrubber-checker-postgres@.service.erb
@@ -17,10 +17,11 @@
Environment=SWH_MAIN_PACKAGE=<%= @sentry_swh_package %>
<%- end -%>
Environment=SWH_SCRUBBER_CLI_EXTRA_ARGS="--help"
+Environment=SWH_CONFIG_FILENAME=
User=<%= @user %>
Group=<%= @group %>
Type=simple
-ExecStart=/usr/bin/swh scrubber --config-file <%= @config_file %> check storage $SWH_SCRUBBER_CLI_EXTRA_ARGS
+ExecStart=/usr/bin/swh scrubber check storage $SWH_SCRUBBER_CLI_EXTRA_ARGS
[Install]
WantedBy=multi-user.target

File Metadata

Mime Type
text/plain
Expires
Wed, Jul 2, 11:00 AM (1 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3215922

Event Timeline

About · Developer information · Legal