Page MenuHomeSoftware Heritage
Files F9346041

D7118.id25844.diff
No OneTemporary
Actions

D7118.id25844.diff
View Options

diff --git a/data/common/common.yaml b/data/common/common.yaml
--- a/data/common/common.yaml
+++ b/data/common/common.yaml
@@ -3657,3 +3657,42 @@
postgresql::globals::version: "%{lookup('swh::postgresql::version')}"
docker::compose::version: "1.29.2"
+
+sanoid::templates:
+ pg_backup:
+ frequent_period: 0
+ pre_snapshot_script: /usr/local/bin/start_pg_backup.sh
+ post_snapshot_script: /usr/local/bin/stop_pg_backup.sh
+ script_timeout: 60
+ monthly: 3
+ daily: 30
+ hourly: 0
+ frequently: 0
+ autoprune: yes
+
+ pg_wal_backup:
+ frequent_period: 0
+ monthly: 3
+ daily: 30
+ hourly: 0
+ frequently: 0
+ autoprune: yes
+
+ backup: # from the default sanoid configuration
+ autoprune: yes
+ frequently: 0
+ hourly: 30
+ daily: 30
+ monthly: 3
+ yearly: 0
+ ### don't take new snapshots - snapshots on backup
+ ### datasets are replicated in from source, not
+ ### generated locally
+ autosnap: no
+ ### monitor hourlies and dailies, but don't warn or
+ ### crit until they're over 48h old, since replication
+ ### is typically daily only
+ hourly_warn: 2880
+ hourly_crit: 3600
+ daily_warn: 48
+ daily_crit: 60
diff --git a/data/hostname/dali.internal.admin.swh.network.yaml b/data/hostname/dali.internal.admin.swh.network.yaml
--- a/data/hostname/dali.internal.admin.swh.network.yaml
+++ b/data/hostname/dali.internal.admin.swh.network.yaml
@@ -24,3 +24,26 @@
prometheus::sql::config_snippets:
- activity
- queries
+
+sanoid::configuration:
+
+ storage_config:
+ data/postgresql:
+ use_template: backup
+ data/postgresql/wal:
+ use_template: backup
+
+ local_config:
+ data/postgresql:
+ use_template: pg_backup
+ hourly: 0
+ monthly: 1
+ daily: 7
+ data/postgresql/wal:
+ use_template: pg_wal_backup
+ hourly: 0
+ monthly: 1
+ daily: 7
+
+
+
diff --git a/site-modules/profile/files/sanoid/start_pg_backup.sh b/site-modules/profile/files/sanoid/start_pg_backup.sh
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/files/sanoid/start_pg_backup.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# File managed by puppet. All modifications will be lost.
+
+set -ex
+
+BACKUP_NAME="${SANOID_SNAPNAME:-backup}"
+
+sudo -i -u postgres psql -c "select pg_start_backup('$BACKUP_NAME', true)"
diff --git a/site-modules/profile/files/sanoid/stop_pg_backup.sh b/site-modules/profile/files/sanoid/stop_pg_backup.sh
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/files/sanoid/stop_pg_backup.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# File managed by puppet. All modifications will be lost.
+
+## - stop the postgresql backup
+## - replace the wal snapshot if it was taken before the postgresql snapshot
+## to ensure all the needed wals are present
+
+set -ex
+DATASET="${SANOID_TARGET}"
+SNAPSHOT_NAME="${SANOID_SNAPNAME:-backup}"
+
+echo "$0 start"
+
+sudo -i -u postgres psql -c "select pg_stop_backup()"
+
+echo "Testing wal shapshot to ensure it is posterior"
+# as sanoid does not guaranty the snapshot orders
+
+if [ -n "${DATASET}" ]; then
+ WAL_DATASET="${DATASET}/wal" # by convention
+ FULL_SNAPSHOT_NAME="${WAL_DATASET}@${SNAPSHOT_NAME}"
+ if zfs list -t snapshot "${FULL_SNAPSHOT_NAME}"; then
+ zfs destroy "${FULL_SNAPSHOT_NAME}"
+ zfs snapshot "${FULL_SNAPSHOT_NAME}"
+ fi
+else
+ echo "Dataset name not set"
+ exit 1
+fi
+
+echo "$0 done"
diff --git a/site-modules/profile/manifests/sanoid/backup.pp b/site-modules/profile/manifests/sanoid/backup.pp
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/manifests/sanoid/backup.pp
@@ -0,0 +1,43 @@
+class profile::sanoid::backup {
+ ensure_packages('sanoid')
+
+ $config_dir = '/etc/sanoid'
+ $config_file = "${config_dir}/sanoid.conf"
+ $host_configuration = lookup('sanoid::configuration')
+ $dataset_configuration = $host_configuration["local_config"]
+ $sanoid_templates = lookup('sanoid::templates')
+
+ file {$config_dir:
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ }
+
+ # template uses $dataset_configuration and $sanoid_templates
+ file {$config_file:
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0644',
+ content => template('profile/sanoid/sanoid.conf.erb'),
+ require => File[$config_dir],
+ }
+
+ file {'/usr/local/bin/start_pg_backup.sh':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0744',
+ source => 'puppet:///modules/profile/sanoid/start_pg_backup.sh'
+ }
+ file {'/usr/local/bin/stop_pg_backup.sh':
+ ensure => present,
+ owner => root,
+ group => root,
+ mode => '0744',
+ source => 'puppet:///modules/profile/sanoid/stop_pg_backup.sh'
+ }
+
+}
+
diff --git a/site-modules/profile/templates/sanoid/sanoid.conf.erb b/site-modules/profile/templates/sanoid/sanoid.conf.erb
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/templates/sanoid/sanoid.conf.erb
@@ -0,0 +1,17 @@
+# File managed with puppet (module profile::sanoid::backup)
+# All modifications will be lost
+
+<%- @dataset_configuration.each do |dataset, config| -%>
+[<%= dataset %>]
+<%- config.each do | key, value | -%>
+ <%= key %> = <%= value %>
+<%- end -%>
+<% end %>
+
+# Templates
+<%- @sanoid_templates.each do |template, config| -%>
+[template_<%= template %>]
+<%- config.each do | key, value | -%>
+ <%= key %> = <%= value %>
+<%- end -%>
+<% end %>
diff --git a/site-modules/role/manifests/swh_admin_database.pp b/site-modules/role/manifests/swh_admin_database.pp
--- a/site-modules/role/manifests/swh_admin_database.pp
+++ b/site-modules/role/manifests/swh_admin_database.pp
@@ -2,4 +2,5 @@
include profile::postgresql
include profile::postgresql::server
include profile::prometheus::sql
+ include profile::sanoid::backup
}

File Metadata

Mime Type
text/plain
Expires
Thu, Jul 3, 3:41 PM (2 w, 10 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3225152

Event Timeline

About · Developer information · Legal