diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -1860,10 +1860,6 @@ swh::deploy::deposit::e2e:archive: /usr/share/swh/icinga-plugins/data/deposit/jesuisgpl.tgz swh::deploy::deposit::e2e:metadata: /usr/share/swh/icinga-plugins/data/deposit/jesuisgpl.tgz.xml -# e2e checks on vault -swh::deploy::vault::e2e::storage: "http://saam.internal.softwareheritage.org:%{hiera('swh::remote_service::storage::port')}" -swh::deploy::vault::e2e::webapp: "https://archive.softwareheritage.org" - swh::deploy::deposit::sentry_swh_package: swh.deposit swh::deploy::deposit::sentry_environment: "%{alias('swh::deploy::environment')}" swh::deploy::deposit::sentry_dsn: "https://%{lookup('swh::deploy::deposit::sentry_token')}@sentry.softwareheritage.org/12" diff --git a/data/deployments/production/common.yaml b/data/deployments/production/common.yaml --- a/data/deployments/production/common.yaml +++ b/data/deployments/production/common.yaml @@ -20,3 +20,7 @@ swh::postgresql::shared_buffers: 4GB swh::deploy::search::index: origin-production + +# e2e checks on vault +swh::deploy::vault::e2e::storage: "http://saam.internal.softwareheritage.org:%{hiera('swh::remote_service::storage::port')}" +swh::deploy::vault::e2e::webapp: "https://archive.softwareheritage.org" diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -48,10 +48,12 @@ #### Storage/Indexer/Vault/Scheduler services to use in staging area +swh::remote_service::storage0::url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/" + swh::remote_service::storage::config::storage0: cls: remote args: - url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/" + url: "%{alias('swh::remote_service::storage0::url')}" swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::storage::config::writable: &swh_remote_service_storage_config_writable "%{alias('swh::remote_service::storage::config::storage0')}" @@ -308,3 +310,6 @@ swh::deploy::search::journal_client::service_types: - objects - indexed + +swh::deploy::vault::e2e::storage: "%{alias('swh::remote_service::storage0::url')}" +swh::deploy::vault::e2e::webapp: "https://webapp.staging.swh.network" diff --git a/site-modules/profile/manifests/icinga2/master.pp b/site-modules/profile/manifests/icinga2/master.pp --- a/site-modules/profile/manifests/icinga2/master.pp +++ b/site-modules/profile/manifests/icinga2/master.pp @@ -63,21 +63,26 @@ target => "/etc/icinga2/zones.d/${zonename}/${::fqdn}.conf", } - ::icinga2::object::service {'check-deposit': - import => ['generic-service-check-e2e'], - service_name => 'Check deposit end-to-end', - check_command => 'check-deposit-cmd', - target => "/etc/icinga2/zones.d/${zonename}/${::fqdn}.conf", - host_name => "${::fqdn}", + each(["production", "staging"]) | $env | { + ::icinga2::object::service {"${env}-check-vault": + import => ['generic-service-check-e2e'], + service_name => "(${env}) Check vault end-to-end", + check_command => "${env}-vault-check-cmd", + target => "/etc/icinga2/zones.d/${zonename}/${::fqdn}.conf", + host_name => "${::fqdn}", + } + + ::icinga2::object::service {"${env}-check-deposit": + import => ['generic-service-check-e2e'], + service_name => "(${env}) Check deposit end-to-end", + check_command => "${env}-deposit-check-cmd", + target => "/etc/icinga2/zones.d/${zonename}/${::fqdn}.conf", + host_name => "${::fqdn}", + } } - ::icinga2::object::service {'check-vault': - import => ['generic-service-check-e2e'], - service_name => 'Check vault end-to-end', - check_command => 'check-vault-cmd', - target => "/etc/icinga2/zones.d/${zonename}/${::fqdn}.conf", - host_name => "${::fqdn}", - } + ::Profile::Icinga2::Objects::E2e_checks_deposit <<| |>> + ::Profile::Icinga2::Objects::E2e_checks_vault <<| |>> ::Icinga2::Object::Host <<| |>> ::Icinga2::Object::Endpoint <<| |>> diff --git a/site-modules/profile/manifests/icinga2/objects.pp b/site-modules/profile/manifests/icinga2/objects.pp --- a/site-modules/profile/manifests/icinga2/objects.pp +++ b/site-modules/profile/manifests/icinga2/objects.pp @@ -11,7 +11,6 @@ include profile::icinga2::objects::static_checks include profile::icinga2::objects::agent_checks include profile::icinga2::objects::exported_checks - include profile::icinga2::objects::e2e_checks include profile::icinga2::plugins::rabbitmq diff --git a/site-modules/profile/manifests/icinga2/objects/e2e_checks.pp b/site-modules/profile/manifests/icinga2/objects/e2e_checks.pp deleted file mode 100644 --- a/site-modules/profile/manifests/icinga2/objects/e2e_checks.pp +++ /dev/null @@ -1,55 +0,0 @@ -# swh's end-to-end checks on the icinga master -class profile::icinga2::objects::e2e_checks { - $checks_file = '/etc/icinga2/conf.d/e2e-checks.conf' - - $deposit_server = lookup('swh::deploy::deposit::e2e::server') - $deposit_user = lookup('swh::deploy::deposit::e2e::user') - $deposit_pass = lookup('swh::deploy::deposit::e2e::password') - $deposit_collection = lookup('swh::deploy::deposit::e2e::collection') - $deposit_poll_interval = lookup('swh::deploy::deposit::e2e::poll_interval') - $deposit_archive = lookup('swh::deploy::deposit::e2e:archive') - $deposit_metadata = lookup('swh::deploy::deposit::e2e:metadata') - - $server_vault = lookup('swh::deploy::vault::e2e::storage') - $server_webapp = lookup('swh::deploy::vault::e2e::webapp') - - $packages = ['python3-swh.icingaplugins'] - - package {$packages: - ensure => present - } - - ::icinga2::object::checkcommand {'check-deposit-cmd': - import => ['plugin-check-command'], - command => [ - '/usr/bin/swh', 'icinga_plugins', - '-w', '600', - 'check-deposit', - '--server', $deposit_server, - '--username', $deposit_user, - '--password', $deposit_pass, - '--collection', $deposit_collection, - '--poll-interval', $deposit_poll_interval, - 'single', - '--archive', $deposit_archive, - '--metadata', $deposit_metadata, - ], - # XXX: Should probably be split into usual commands with arguments - # arguments => ... - timeout => 900, - target => $checks_file, - require => Package[$packages] - } - - ::icinga2::object::checkcommand {'check-vault-cmd': - import => ['plugin-check-command'], - command => [ - '/usr/bin/swh', 'icinga_plugins', 'check-vault', - '--swh-storage-url', $server_vault, - '--swh-web-url', $server_webapp, - 'directory' - ], - target => $checks_file, - require => Package[$packages] - } -} diff --git a/site-modules/profile/manifests/icinga2/objects/e2e_checks_base.pp b/site-modules/profile/manifests/icinga2/objects/e2e_checks_base.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/icinga2/objects/e2e_checks_base.pp @@ -0,0 +1,8 @@ +# swh's end-to-end checks common behavior +class profile::icinga2::objects::e2e_checks_base { + $packages = ['python3-swh.icingaplugins'] + package {$packages: + ensure => present + } + $check_file => '/etc/icinga2/conf.d/e2e-checks.conf' +} diff --git a/site-modules/profile/manifests/icinga2/objects/e2e_checks_deposit.pp b/site-modules/profile/manifests/icinga2/objects/e2e_checks_deposit.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/icinga2/objects/e2e_checks_deposit.pp @@ -0,0 +1,37 @@ +# swh deposit end-to-end checks on the icinga master +define profile::icinga2::objects::e2e_checks_deposit ( + String $check_command = $name, + String $deposit_server, + String $deposit_user, + String $deposit_pass, + String $deposit_collection, + Integer $deposit_poll_interval, + String $deposit_archive, + String $deposit_metadata, + String $environment, +) { + include ::profile::icinga2::objects::e2e_checks_base + + ::icinga2::object::checkcommand {$check_command: + import => ['plugin-check-command'], + command => [ + '/usr/bin/swh', 'icinga_plugins', + '-w', '600', + 'check-deposit', + '--server', $deposit_server, + '--username', $deposit_user, + '--password', $deposit_pass, + '--collection', $deposit_collection, + '--poll-interval', $deposit_poll_interval, + 'single', + '--archive', $deposit_archive, + '--metadata', $deposit_metadata, + ], + # XXX: Should probably be split into usual commands with arguments + # arguments => ... + timeout => 900, + target => $::profile::icinga2::objects::e2e_checks_base::packages::check_file, + require => Package[$::profile::icinga2::objects::e2e_checks_base::packages], + } + +} diff --git a/site-modules/profile/manifests/icinga2/objects/e2e_checks_vault.pp b/site-modules/profile/manifests/icinga2/objects/e2e_checks_vault.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/icinga2/objects/e2e_checks_vault.pp @@ -0,0 +1,21 @@ +# swh vault end-to-end checks on the icinga master +define profile::icinga2::objects::e2e_checks_vault ( + String $check_command = $name, + String $server_vault, + String $server_webapp, + String $environment, +) { + include ::profile::icinga2::objects::e2e_checks_base + + ::icinga2::object::checkcommand {$check_command: + import => ['plugin-check-command'], + command => [ + '/usr/bin/swh', 'icinga_plugins', 'check-vault', + '--swh-storage-url', $server_vault, + '--swh-web-url', $server_webapp, + 'directory' + ], + target => $::profile::icinga2::objects::e2e_checks_base::packages::check_file, + require => Package[$::profile::icinga2::objects::e2e_checks_base::packages], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/deposit.pp b/site-modules/profile/manifests/swh/deploy/deposit.pp --- a/site-modules/profile/manifests/swh/deploy/deposit.pp +++ b/site-modules/profile/manifests/swh/deploy/deposit.pp @@ -207,6 +207,17 @@ } } + # Install deposit end-to-end checks + @@profile::icinga2::objects::e2e_checks_deposit {"${environment}-deposit-check-cmd": + deposit_server => lookup('swh::deploy::deposit::e2e::server'), + deposit_user => lookup('swh::deploy::deposit::e2e::user'), + deposit_pass => lookup('swh::deploy::deposit::e2e::password'), + deposit_collection => lookup('swh::deploy::deposit::e2e::collection'), + deposit_poll_interval => lookup('swh::deploy::deposit::e2e::poll_interval'), + deposit_archive => lookup('swh::deploy::deposit::e2e:archive'), + deposit_metadata => lookup('swh::deploy::deposit::e2e:metadata'), + environment => $environment, + } include profile::filebeat # To remove when cleanup is done diff --git a/site-modules/profile/manifests/swh/deploy/vault.pp b/site-modules/profile/manifests/swh/deploy/vault.pp --- a/site-modules/profile/manifests/swh/deploy/vault.pp +++ b/site-modules/profile/manifests/swh/deploy/vault.pp @@ -2,10 +2,12 @@ class profile::swh::deploy::vault { include ::profile::swh::deploy::base_vault + Package['python3-swh.vault'] ~> Service['gunicorn-swh-vault'] $user = lookup('swh::deploy::vault::user') $cache_directory = lookup('swh::deploy::vault::cache') + file {$cache_directory: ensure => directory, owner => $user, @@ -17,4 +19,11 @@ executable => 'swh.vault.api.server:make_app_from_configfile()', worker => 'async', } + + # Install vault end-to-end checks + @@profile::icinga2::objects::e2e_checks_vault {"${environment}-check-vault-cmd": + server_vault => lookup('swh::deploy::vault::e2e::storage'), + server_webapp => lookup('swh::deploy::vault::e2e::webapp'), + environment => $environment, + } }