Page MenuHomeSoftware Heritage
Files F7066431

D7118.id25828.diff
No OneTemporary
Actions

D7118.id25828.diff
View Options

diff --git a/data/common/common.yaml b/data/common/common.yaml
--- a/data/common/common.yaml
+++ b/data/common/common.yaml
@@ -3657,3 +3657,42 @@
postgresql::globals::version: "%{lookup('swh::postgresql::version')}"
docker::compose::version: "1.29.2"
+
+sanoid::templates:
+ pg_backup:
+ frequent_period: 0
+ pre_snapshot_script: /usr/local/bin/start_pg_backup.sh
+ post_snapshot_script: /usr/local/bin/stop_pg_backup.sh
+ script_timeout: 60
+ monthly: 3
+ daily: 30
+ hourly: 0
+ frequently: 0
+ autoprune: yes
+
+ pg_wal_backup:
+ frequent_period: 0
+ monthly: 3
+ daily: 30
+ hourly: 0
+ frequently: 0
+ autoprune: yes
+
+ backup: # from the default sanoid configuration
+ autoprune: yes
+ frequently: 0
+ hourly: 30
+ daily: 30
+ monthly: 3
+ yearly: 0
+ ### don't take new snapshots - snapshots on backup
+ ### datasets are replicated in from source, not
+ ### generated locally
+ autosnap: no
+ ### monitor hourlies and dailies, but don't warn or
+ ### crit until they're over 48h old, since replication
+ ### is typically daily only
+ hourly_warn: 2880
+ hourly_crit: 3600
+ daily_warn: 48
+ daily_crit: 60
diff --git a/data/hostname/dali.internal.admin.swh.network.yaml b/data/hostname/dali.internal.admin.swh.network.yaml
--- a/data/hostname/dali.internal.admin.swh.network.yaml
+++ b/data/hostname/dali.internal.admin.swh.network.yaml
@@ -24,3 +24,26 @@
prometheus::sql::config_snippets:
- activity
- queries
+
+sanoid::configuration:
+
+ storage_config:
+ data/postgresql:
+ use_template: backup
+ data/postgresql/wal:
+ use_template: backup
+
+ local_config:
+ data/postgresql:
+ use_template: pg_backup
+ hourly: 0
+ monthly: 1
+ daily: 7
+ data/postgresql/wal:
+ use_template: pg_wal_backup
+ hourly: 0
+ monthly: 1
+ daily: 7
+
+
+
diff --git a/site-modules/profile/files/sanoid/start_pg_backup.sh b/site-modules/profile/files/sanoid/start_pg_backup.sh
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/files/sanoid/start_pg_backup.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# File managed by puppet. All modifications will be lost.
+
+set -ex
+
+BACKUP_NAME="${SANOID_SNAPNAME:-backup}"
+
+sudo -i -u postgres psql -c "select pg_start_backup('$BACKUP_NAME', true)"
diff --git a/site-modules/profile/files/sanoid/stop_pg_backup.sh b/site-modules/profile/files/sanoid/stop_pg_backup.sh
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/files/sanoid/stop_pg_backup.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# File managed by puppet. All modifications will be lost.
+
+## - stop the postgresql backup
+## - replace the wal snapshot if it was taken before the postgresql snapshot
+## to ensure all the needed wals are present
+
+set -ex
+DATASET="${SANOID_TARGET}"
+SNAPSHOT_NAME="${SANOID_SNAPNAME:-backup}"
+
+echo "$0 start"
+
+sudo -i -u postgres psql -c "select pg_stop_backup()"
+
+echo "Testing wal shapshot to ensure it is posterior"
+# as sanoid does not guaranty the snapshot orders
+
+if [ -n "${DATASET}" ]; then
+ WAL_DATASET="${DATASET}/wal" # by convention
+ FULL_SNAPSHOT_NAME="${WAL_DATASET}@${SNAPSHOT_NAME}"
+ if zfs list -t snapshot "${FULL_SNAPSHOT_NAME}"; then
+ zfs destroy "${FULL_SNAPSHOT_NAME}"
+ zfs snapshot "${FULL_SNAPSHOT_NAME}"
+ fi
+else
+ echo "Dataset name not set"
+ exit 1
+fi
+
+echo "$0 done"
diff --git a/site-modules/profile/manifests/sanoid/backup.pp b/site-modules/profile/manifests/sanoid/backup.pp
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/manifests/sanoid/backup.pp
@@ -0,0 +1,73 @@
+class profile::sanoid::backup {
+ ensure_packages('sanoid')
+
+ $config_dir = '/etc/sanoid'
+ $config_file = "${config_dir}/sanoid.conf"
+ $host_configuration = lookup('sanoid::configuration')
+ $sanoid_templates = lookup('sanoid::templates')
+
+ file {$config_dir:
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ }
+
+ concat {$config_file:
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0644',
+ ensure_newline => true,
+ require => File[$config_dir]
+ }
+
+ concat::fragment {"${config_file}:header":
+ target => $config_file,
+ content => "# File managed with puppet (module profile::varnish)\n# All modifications will be lost\n\n",
+ order => '00',
+ }
+
+ $host_configuration["local_config"].each | $dataset, $config | {
+ $toml_config = "${ to_toml({$dataset => $config }) }\n"
+ # unescape the toml generated by to_toml
+ # sanoid does not espect escaped strings
+ # ["data/postgresql"] => [data/postgreql]
+ $unescaped_toml = regsubst($toml_config, '"', '', 'G')
+ concat::fragment {"${config_file}:${dataset}":
+ target => $config_file,
+ content => $unescaped_toml,
+ order => '01',
+ }
+ }
+
+ $sanoid_templates.each | $template, $config | {
+ $key = "template_${template}"
+ # unescape the toml generated by to_toml
+ # sanoid does not espect escaped strings
+ $toml_config = "${ to_toml({$key => $config }) }\n"
+ $unescaped_toml = regsubst($toml_config, '"', '', 'G')
+ concat::fragment {"template_${config_file}:${template}":
+ target => $config_file,
+ content => $unescaped_toml,
+ order => '99',
+ }
+ }
+
+ file {'/usr/local/bin/start_pg_backup.sh':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0744',
+ source => 'puppet:///modules/profile/sanoid/start_pg_backup.sh'
+ }
+ file {'/usr/local/bin/stop_pg_backup.sh':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0744',
+ source => 'puppet:///modules/profile/sanoid/stop_pg_backup.sh'
+ }
+
+}
+
diff --git a/site-modules/role/manifests/swh_admin_database.pp b/site-modules/role/manifests/swh_admin_database.pp
--- a/site-modules/role/manifests/swh_admin_database.pp
+++ b/site-modules/role/manifests/swh_admin_database.pp
@@ -2,4 +2,5 @@
include profile::postgresql
include profile::postgresql::server
include profile::prometheus::sql
+ include profile::sanoid::backup
}

File Metadata

Mime Type
text/plain
Expires
Nov 5 2024, 8:38 AM (12 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3225158

Event Timeline

About · Developer information · Legal