Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9697258
D7590.id27485.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
9 KB
Subscribers
None
D7590.id27485.diff
View Options
diff --git a/data/common/common.yaml b/data/common/common.yaml
--- a/data/common/common.yaml
+++ b/data/common/common.yaml
@@ -1747,9 +1747,10 @@
swh::deploy::journal::backfill::user: swhstorage
swh::deploy::journal::backfill::group: swhstorage
+swh::deploy::storage::db::config::read-only: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=guest dbname=%{hiera('swh::deploy::storage::db::dbname')} password=guest"
swh::deploy::storage::config::local-read-only:
cls: local
- db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=guest dbname=%{hiera('swh::deploy::storage::db::dbname')} password=guest"
+ db: "%{alias('swh::deploy::storage::db::config::read-only')}"
# objstorage is required but it's not used in that context
objstorage:
cls: memory
@@ -1888,6 +1889,38 @@
swh::deploy::journal::prefix: swh.journal.objects
+swh::deploy::scrubber::sentry_swh_package: swh.scrubber
+swh::deploy::scrubber::sentry_environment: "%{alias('swh::deploy::environment')}"
+swh::deploy::scrubber::sentry_dsn: "https://%{lookup('swh::deploy::scrubber::sentry_token')}@sentry.softwareheritage.org/23"
+
+swh::deploy::scrubber::checker::postgres::conf_directory: "%{hiera('swh::conf_directory')}/scrubber"
+swh::deploy::scrubber::checker::postgres::conf_file: "%{hiera('swh::deploy::scrubber::checker::postgres::conf_directory')}/postgres.yml"
+swh::deploy::scrubber::checker::postgres::user: swhworker
+swh::deploy::scrubber::checker::postgres::group: swhdev
+swh::deploy::scrubber::checker::postgres::object_types:
+ # - content
+ # - directory
+ # - revision
+ - release
+
+swh::deploy::scrubber::checker::postgres::ranges:
+ - 00000000000000000000:44444444444444444444
+ - 44444444444444444444:88888888888888888888
+ - 88888888888888888888:bbbbbbbbbbbbbbbbbbbb
+ - bbbbbbbbbbbbbbbbbbbb:ffffffffffffffffffff
+
+# password entry in private-data
+swh::deploy::scrubber::db::config: "host=%{hiera('swh::deploy::scrubber::db::host')} port=%{hiera('swh::deploy::db::pgbouncer::port')} dbname=%{hiera('swh::deploy::scrubber::db::dbname')} user=%{hiera('swh::deploy::scrubber::db::user')} password=%{hiera('swh::deploy::scrubber::db::password')}"
+
+swh::deploy::scrubber::checker::postgres::config:
+ scrubber_db:
+ cls: local
+ db: "%{alias('swh::deploy::scrubber::db::config')}"
+ storage:
+ cls: postgresql
+ db: "%{alias('swh::deploy::storage::db::config::read-only')}"
+ objstorage: "%{alias('swh::remote_service::objstorage::config')}"
+
swh::deploy::journal_simple_checker_producer::conf_file: "%{hiera('swh::deploy::journal::conf_directory')}/checker.yml"
swh::deploy::journal_simple_checker_producer::user: swhstorage
swh::deploy::journal_simple_checker_producer::group: swhstorage
@@ -3769,3 +3802,4 @@
sonatype: http://oss.sonatype.org/content/repositories/releases/
jboss: https://repository.jboss.org/maven2/
atlassian-public: https://maven.atlassian.com/public/
+
diff --git a/data/subnets/vagrant.yaml b/data/subnets/vagrant.yaml
--- a/data/subnets/vagrant.yaml
+++ b/data/subnets/vagrant.yaml
@@ -239,6 +239,8 @@
host: worker3.internal.staging.swh.network
10.168.130.110:
host: objstorage0.internal.staging.swh.network
+ 10.168.130.120:
+ host: scrubber0.internal.staging.swh.network
10.168.130.160:
host: mirror-test.internal.staging.swh.network
10.168.200.22:
diff --git a/manifests/site.pp b/manifests/site.pp
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -183,6 +183,10 @@
include role::swh_counters_with_journal_client
}
+node /^scrubber\d+\.internal\.staging\.swh\.network/ {
+ include role::swh_scrubber_checkers
+}
+
node 'webapp.internal.staging.swh.network' {
include role::swh_webapp
}
diff --git a/site-modules/profile/files/swh/deploy/scrubber/swh-scrubber.slice b/site-modules/profile/files/swh/deploy/scrubber/swh-scrubber.slice
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/files/swh/deploy/scrubber/swh-scrubber.slice
@@ -0,0 +1,5 @@
+[Unit]
+Description=SWH Scrubber services
+
+[Slice]
+MemoryMax=10%
diff --git a/site-modules/profile/manifests/swh/deploy/scrubber/checker/postgres.pp b/site-modules/profile/manifests/swh/deploy/scrubber/checker/postgres.pp
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/manifests/swh/deploy/scrubber/checker/postgres.pp
@@ -0,0 +1,90 @@
+# Deployment of the swh.scrubber's checker postgres service
+
+class profile::swh::deploy::scrubber::checker::postgres {
+ $sentry_dsn = lookup("swh::deploy::scrubber::sentry_dsn")
+ $sentry_environment = lookup("swh::deploy::scrubber::sentry_environment")
+ $sentry_swh_package = lookup("swh::deploy::scrubber::sentry_swh_package")
+
+ $config_dir = lookup('swh::deploy::scrubber::checker::postgres::conf_directory')
+ $config_file = lookup('swh::deploy::scrubber::checker::postgres::conf_file')
+ $config_dict = lookup('swh::deploy::scrubber::checker::postgres::config')
+ $user = lookup('swh::deploy::scrubber::checker::postgres::user')
+ $group = lookup('swh::deploy::scrubber::checker::postgres::group')
+
+ $object_types = lookup('swh::deploy::scrubber::checker::postgres::object_types')
+ $ranges = lookup('swh::deploy::scrubber::checker::postgres::ranges')
+
+ $packages = ['python3-swh.scrubber']
+ ensure_packages($packages)
+
+ file {$config_dir:
+ ensure => 'directory',
+ owner => $user,
+ group => $group,
+ }
+
+ file {$config_file:
+ ensure => present,
+ owner => $user,
+ group => $group,
+ mode => '0640',
+ content => inline_template("<%= @config_dict.to_yaml %>\n"),
+ require => File[$config_dir]
+ }
+
+ $systemd_slice_name = "swh-scrubber.slice"
+ ::systemd::unit_file {$systemd_slice_name:
+ ensure => 'present',
+ source => "puppet:///modules/profile/swh/deploy/scrubber/${systemd_slice_name}",
+ }
+
+ $template_name = 'swh-scrubber-checker-postgres'
+ $template_unit_name = "${template_name}@.service"
+ # Template uses:
+ # - $user
+ # - $group
+ # - $sentry_dsn
+ # - $sentry_environment
+ # - $sentry_swh_package
+ # - $config_file
+ ::systemd::unit_file {$template_unit_name:
+ ensure => present,
+ content => template("profile/swh/deploy/scrubber/${template_unit_name}.erb"),
+ enable => false,
+ require => [
+ File[$config_file],
+ ::Systemd::Unit_file[$systemd_slice_name],
+ Package[$packages],
+ ]
+ }
+
+ $object_types.each | $object_type | {
+ $ranges.each | $range_index, $range | {
+ $ranges_list = $range.split(':')
+ $start_object = $ranges_list[0]
+ $end_object = $ranges_list[1]
+ $service_name = "${template_name}@${object_type}-${range_index}.service"
+
+ $parameters_conf_name = "${service_name}.d/parameters.conf"
+ # Template uses:
+ # - $object_type
+ # - $start_object
+ # - $end_object
+ ::systemd::dropin_file {$parameters_conf_name:
+ ensure => present,
+ unit => $service_name,
+ filename => 'parameters.conf',
+ content => template("profile/swh/deploy/scrubber/parameters.conf.erb"),
+ }
+
+ service {$service_name:
+ ensure => running,
+ enable => false,
+ require => [
+ ::Systemd::Unit_file[$template_unit_name],
+ ::Systemd::Dropin_File[$parameters_conf_name],
+ ],
+ }
+ }
+ }
+}
diff --git a/site-modules/profile/templates/swh/deploy/scrubber/parameters.conf.erb b/site-modules/profile/templates/swh/deploy/scrubber/parameters.conf.erb
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/templates/swh/deploy/scrubber/parameters.conf.erb
@@ -0,0 +1,5 @@
+# Managed by puppet class profile::swh::deploy::scrubber::checker::postgres
+# Modifications will be overwritten
+
+[Service]
+Environment=SWH_SCRUBBER_CLI_EXTRA_ARGS="--object-type <%= @object_type %> --start-object <%= @start_object %> --end-object <%= @end_object %>"
diff --git a/site-modules/profile/templates/swh/deploy/scrubber/swh-scrubber-checker-postgres@.service.erb b/site-modules/profile/templates/swh/deploy/scrubber/swh-scrubber-checker-postgres@.service.erb
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/templates/swh/deploy/scrubber/swh-scrubber-checker-postgres@.service.erb
@@ -0,0 +1,26 @@
+# Scrubber checker postgres template unit file
+# Managed by puppet class profile::swh::deploy::scrubber::checker::postgres
+# Changes will be overwritten
+
+[Unit]
+Description=Software Heritage Scrubber Checker Postgres %i
+After=network.target
+
+[Service]
+<%- if !@sentry_dsn.nil? and !@sentry_dsn.empty? -%>
+Environment=SWH_SENTRY_DSN=<%= @sentry_dsn %>
+<%- end -%>
+<%- if !@sentry_environment.nil? and !@sentry_environment.empty? -%>
+Environment=SWH_SENTRY_ENVIRONMENT=<%= @sentry_environment %>
+<%- end -%>
+<%- if !@sentry_swh_package.nil? and !@sentry_swh_package.empty? -%>
+Environment=SWH_MAIN_PACKAGE=<%= @sentry_swh_package %>
+<%- end -%>
+Environment=SWH_SCRUBBER_CLI_EXTRA_ARGS="--help"
+User=<%= @user %>
+Group=<%= @group %>
+Type=simple
+ExecStart=/usr/bin/swh scrubber --config-file <%= @config_file %> check storage ${SWH_SCRUBBER_CLI_EXTRA_ARGS}
+
+[Install]
+WantedBy=multi-user.target
diff --git a/site-modules/role/manifests/swh_scrubber_checkers.pp b/site-modules/role/manifests/swh_scrubber_checkers.pp
new file mode 100644
--- /dev/null
+++ b/site-modules/role/manifests/swh_scrubber_checkers.pp
@@ -0,0 +1,4 @@
+class role::swh_scrubber_checkers inherits role::swh_base {
+ # checker(s)
+ include profile::swh::deploy::scrubber::checker::postgres
+}
diff --git a/site-modules/role/manifests/swh_storage_with_journal.pp b/site-modules/role/manifests/swh_storage_with_journal.pp
--- a/site-modules/role/manifests/swh_storage_with_journal.pp
+++ b/site-modules/role/manifests/swh_storage_with_journal.pp
@@ -2,7 +2,7 @@
include profile::postgresql::client
include profile::swh::deploy::journal::backfill
- # journal
+ # journal
include profile::zookeeper
include profile::kafka::broker
}
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Sun, Aug 17, 11:15 PM (1 w, 1 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3218220
Attached To
D7590: Deploy swh-scrubber-checker-services to scrubber0.staging
Event Timeline
Log In to Comment