diff --git a/site-modules/profile/manifests/swh/deploy/deposit.pp b/site-modules/profile/manifests/swh/deploy/deposit.pp index b16af739..5b6d2bd1 100644 --- a/site-modules/profile/manifests/swh/deploy/deposit.pp +++ b/site-modules/profile/manifests/swh/deploy/deposit.pp @@ -1,276 +1,214 @@ # Deployment of the swh.deposit server class profile::swh::deploy::deposit { $config_directory = lookup('swh::deploy::deposit::config_directory') $config_file = lookup('swh::deploy::deposit::config_file') $user = lookup('swh::deploy::deposit::user') $group = lookup('swh::deploy::deposit::group') $conf_hiera = lookup('swh::deploy::deposit::config') $static_dir = '/usr/lib/python3/dist-packages/swh/deposit/static' $backend_listen_host = lookup('swh::deploy::deposit::backend::listen::host') $backend_listen_port = lookup('swh::deploy::deposit::backend::listen::port') $backend_listen_address = "${backend_listen_host}:${backend_listen_port}" $backend_workers = lookup('swh::deploy::deposit::backend::workers') $backend_http_keepalive = lookup('swh::deploy::deposit::backend::http_keepalive') $backend_http_timeout = lookup('swh::deploy::deposit::backend::http_timeout') $backend_reload_mercy = lookup('swh::deploy::deposit::backend::reload_mercy') $vhost_url = lookup('swh::deploy::deposit::url') $cert_name = lookup('swh::deploy::deposit::vhost::letsencrypt_cert') $vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains'] $full_conf = $conf_hiera + {allowed_hosts => $vhosts} if $swh_hostname['fqdn'] in $vhosts { $vhost_name = $swh_hostname['fqdn'] } else { $vhost_name = $vhosts[0] } $vhost_aliases = delete($vhosts, $vhost_name) $vhost_port = lookup('apache::http_port') $vhost_docroot = "/var/www/${vhost_name}" $vhost_basic_auth_file = "${config_directory}/http_auth" # swh::deploy::deposit::vhost::basic_auth_content in private $vhost_basic_auth_content = lookup('swh::deploy::deposit::vhost::basic_auth_content') $vhost_ssl_port = lookup('apache::https_port') $vhost_ssl_protocol = lookup('swh::deploy::deposit::vhost::ssl_protocol') $vhost_ssl_honorcipherorder = lookup('swh::deploy::deposit::vhost::ssl_honorcipherorder') $vhost_ssl_cipher = lookup('swh::deploy::deposit::vhost::ssl_cipher') $locked_endpoints = lookup('swh::deploy::deposit::locked_endpoints', Array, 'unique') $media_root_directory = lookup('swh::deploy::deposit::media_root_directory') include ::gunicorn # Install the necessary deps ::profile::swh::deploy::install_web_deps { 'swh-deposit': services => ['gunicorn-swh-deposit'], backport_list => 'swh::deploy::deposit::backported_packages', swh_packages => ['python3-swh.deposit'], ensure => present, } file {$config_directory: ensure => directory, owner => 'root', group => $group, mode => '0755', } # swh's configuration part (upload size, etc...) file {$config_file: ensure => present, owner => 'root', group => $group, mode => '0640', content => inline_template("<%= @full_conf.to_yaml %>\n"), notify => Service['gunicorn-swh-deposit'], } file {$media_root_directory: ensure => directory, owner => $user, group => $group, mode => '2750', } $sentry_dsn = lookup("swh::deploy::deposit::sentry_dsn", Optional[String], 'first', undef) $sentry_environment = lookup("swh::deploy::deposit::sentry_environment", Optional[String], 'first', undef) $sentry_swh_package = lookup("swh::deploy::deposit::sentry_swh_package", Optional[String], 'first', undef) ::gunicorn::instance {'swh-deposit': ensure => enabled, user => $user, group => $group, executable => 'django.core.wsgi:get_wsgi_application()', environment => { 'SWH_CONFIG_FILENAME' => $config_file, 'DJANGO_SETTINGS_MODULE' => 'swh.deposit.settings.production', 'SWH_SENTRY_DSN' => $sentry_dsn, 'SWH_SENTRY_ENVIRONMENT' => $sentry_environment, 'SWH_MAIN_PACKAGE' => $sentry_swh_package, }, settings => { bind => $backend_listen_address, workers => $backend_workers, worker_class => 'sync', timeout => $backend_http_timeout, graceful_timeout => $backend_reload_mercy, keepalive => $backend_http_keepalive, } } $endpoint_directories = $locked_endpoints.map |$endpoint| { { path => "^${endpoint}", provider => 'locationmatch', auth_type => 'Basic', auth_name => 'Software Heritage Deposit', auth_user_file => $vhost_basic_auth_file, auth_require => 'valid-user', } } include ::profile::apache::common include ::apache::mod::proxy include ::apache::mod::headers ::apache::vhost {"${vhost_name}_non-ssl": servername => $vhost_name, serveraliases => $vhost_aliases, port => $vhost_port, docroot => $vhost_docroot, proxy_pass => [ { path => '/static', url => '!', }, { path => '/robots.txt', url => '!', }, { path => '/favicon.ico', url => '!', }, { path => '/', url => "http://${backend_listen_address}/", }, ], directories => [ { path => '/1', provider => 'location', allow => 'from all', satisfy => 'Any', headers => ['add Access-Control-Allow-Origin "*"'], }, { path => $static_dir, options => ['-Indexes'], }, ] + $endpoint_directories, aliases => [ { alias => '/static', path => $static_dir, }, { alias => '/robots.txt', path => "${static_dir}/robots.txt", }, ], require => [ File[$vhost_basic_auth_file], ] } - include ::profile::hitch - realize(::Profile::Hitch::Ssl_cert[$cert_name]) - - include ::profile::varnish - $url_scheme = split($vhost_url, ':')[0] - - if $url_scheme == 'https' { - ::profile::varnish::vhost {$vhost_name: - aliases => $vhost_aliases, - hsts_max_age => lookup('strict_transport_security::max_age'), - } - } - file {$vhost_basic_auth_file: ensure => present, owner => 'root', group => 'www-data', mode => '0640', content => $vhost_basic_auth_content, } $icinga_checks_file = lookup('icinga2::exported_checks::filename') @@::icinga2::object::service {"swh-deposit api (localhost on ${::fqdn})": service_name => 'swh-deposit api (localhost)', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', command_endpoint => $::fqdn, vars => { http_address => '127.0.0.1', http_port => $backend_listen_port, http_uri => '/', http_string => 'The Software Heritage Deposit', }, target => $icinga_checks_file, tag => 'icinga2::exported', } if $backend_listen_host != '127.0.0.1' { @@::icinga2::object::service {"swh-deposit api (remote on ${::fqdn})": service_name => 'swh-deposit api (remote)', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_port => $backend_listen_port, http_uri => '/', http_string => 'The Software Heritage Deposit', }, target => $icinga_checks_file, tag => 'icinga2::exported', } } - @@::icinga2::object::service {"swh-deposit http redirect on ${::fqdn}": - service_name => 'swh deposit http redirect', - import => ['generic-service'], - host_name => $::fqdn, - check_command => 'http', - vars => { - http_address => $vhost_name, - http_vhost => $vhost_name, - http_port => $vhost_port, - http_uri => '/', - }, - target => $icinga_checks_file, - tag => 'icinga2::exported', - } - - @@::icinga2::object::service {"swh-deposit https on ${::fqdn}": - service_name => 'swh deposit', - import => ['generic-service'], - host_name => $::fqdn, - check_command => 'http', - vars => { - http_address => $vhost_name, - http_vhost => $vhost_name, - http_port => $vhost_ssl_port, - http_ssl => true, - http_sni => true, - http_uri => '/', - http_onredirect => sticky - }, - target => $icinga_checks_file, - tag => 'icinga2::exported', - } - - @@::icinga2::object::service {"swh-deposit https certificate ${::fqdn}": - service_name => 'swh deposit https certificate', - import => ['generic-service'], - host_name => $::fqdn, - check_command => 'http', - vars => { - http_address => $vhost_name, - http_vhost => $vhost_name, - http_port => $vhost_ssl_port, - http_ssl => true, - http_sni => true, - http_certificate => 15, - }, - target => $icinga_checks_file, - tag => 'icinga2::exported', - } include profile::filebeat profile::filebeat::log_input { 'deposit-non-ssl-access': paths => [ '/var/log/apache2/deposit.softwareheritage.org_non-ssl_access.log' ], fields => { 'apache_log_type' => 'access_log', }, } } diff --git a/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp b/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp index 3e1b694d..40be2883 100644 --- a/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp +++ b/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp @@ -1,46 +1,89 @@ # Reverse proxy to expose staging services # https://forge.softwareheritage.org/T2747 class profile::swh::deploy::reverse_proxy { include ::profile::hitch include ::profile::varnish $service_names = lookup('swh::deploy::reverse_proxy::services') $varnish_http_port = lookup('varnish::http_port') each($service_names) |$service_name| { # Retrieve certificate name $cert_name = lookup("swh::deploy::${service_name}::vhost::letsencrypt_cert") # Retrieve the list of vhosts $vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains'] if $swh_hostname['fqdn'] in $vhosts { $vhost_name = $swh_hostname['fqdn'] } else { $vhost_name = $vhosts[0] } # Compute aliases, removing the main vhost from the list $vhost_aliases = delete($vhosts, $vhost_name) realize(::Profile::Hitch::Ssl_cert[$cert_name]) ::profile::varnish::vhost {$vhost_name: aliases => $vhost_aliases, hsts_max_age => lookup('strict_transport_security::max_age'), } + $icinga_checks_file = lookup('icinga2::exported_checks::filename') + # icinga alerts - # @@::icinga2::object::service {"${service_name} http redirect on ${::fqdn}": - # service_name => 'swh webapp http redirect', - # import => ['generic-service'], - # host_name => $::fqdn, - # check_command => 'http', - # vars => { - # http_address => $vhost_name, - # http_vhost => $vhost_name, - # http_port => $varnish_http_port, - # http_uri => '/', - # }, - # target => $icinga_checks_file, - # tag => 'icinga2::exported', - # } + @@::icinga2::object::service {"${service_name} http redirect on ${::fqdn}": + service_name => "swh ${service_name} http redirect", + import => ['generic-service'], + host_name => $::fqdn, + check_command => 'http', + vars => { + http_address => $vhost_name, + http_vhost => $vhost_name, + http_port => $varnish_http_port, + http_uri => '/', + }, + target => $icinga_checks_file, + tag => 'icinga2::exported', + } + + $vhost_ssl_port = lookup('apache::https_port') + + # $vhost_ssl_protocol = lookup('swh::deploy::webapp::vhost::ssl_protocol') + # $vhost_ssl_honorcipherorder = lookup('swh::deploy::webapp::vhost::ssl_honorcipherorder') + # $vhost_ssl_cipher = lookup('swh::deploy::webapp::vhost::ssl_cipher') + + @@::icinga2::object::service {"swh-${service_name} https on ${::fqdn}": + service_name => "swh ${service_name}", + import => ['generic-service'], + host_name => $::fqdn, + check_command => 'http', + vars => { + http_address => $vhost_name, + http_vhost => $vhost_name, + http_port => $vhost_ssl_port, + http_ssl => true, + http_sni => true, + http_uri => '/', + http_onredirect => sticky + }, + target => $icinga_checks_file, + tag => 'icinga2::exported', + } + + @@::icinga2::object::service {"swh-${service_name} https certificate ${::fqdn}": + service_name => "swh ${service_name} https certificate", + import => ['generic-service'], + host_name => $::fqdn, + check_command => 'http', + vars => { + http_address => $vhost_name, + http_vhost => $vhost_name, + http_port => $vhost_ssl_port, + http_ssl => true, + http_sni => true, + http_certificate => 15, + }, + target => $icinga_checks_file, + tag => 'icinga2::exported', + } } } diff --git a/site-modules/profile/manifests/swh/deploy/webapp.pp b/site-modules/profile/manifests/swh/deploy/webapp.pp index 40e541e4..3c2cc338 100644 --- a/site-modules/profile/manifests/swh/deploy/webapp.pp +++ b/site-modules/profile/manifests/swh/deploy/webapp.pp @@ -1,291 +1,224 @@ # WebApp deployment class profile::swh::deploy::webapp { $conf_directory = lookup('swh::deploy::webapp::conf_directory') $conf_file = lookup('swh::deploy::webapp::conf_file') $user = lookup('swh::deploy::webapp::user') $group = lookup('swh::deploy::webapp::group') $webapp_config = lookup('swh::deploy::webapp::config') $conf_log_dir = lookup('swh::deploy::webapp::conf::log_dir') $backend_listen_host = lookup('swh::deploy::webapp::backend::listen::host') $backend_listen_port = lookup('swh::deploy::webapp::backend::listen::port') $backend_listen_address = "${backend_listen_host}:${backend_listen_port}" $backend_workers = lookup('swh::deploy::webapp::backend::workers') $backend_http_keepalive = lookup('swh::deploy::webapp::backend::http_keepalive') $backend_http_timeout = lookup('swh::deploy::webapp::backend::http_timeout') $backend_reload_mercy = lookup('swh::deploy::webapp::backend::reload_mercy') $static_dir = '/usr/share/swh/web/static' - $varnish_http_port = lookup('varnish::http_port') - $cert_name = lookup('swh::deploy::webapp::vhost::letsencrypt_cert') $vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains'] $full_webapp_config = $webapp_config + {allowed_hosts => $vhosts} if $swh_hostname['fqdn'] in $vhosts { $vhost_name = $swh_hostname['fqdn'] } else { $vhost_name = $vhosts[0] } $vhost_aliases = delete($vhosts, $vhost_name) $vhost_port = lookup('apache::http_port') $vhost_docroot = "/var/www/${vhost_name}" $vhost_basic_auth_file = "${conf_directory}/http_auth" $vhost_basic_auth_content = lookup('swh::deploy::webapp::vhost::basic_auth_content', String, 'first', '') - $vhost_ssl_port = lookup('apache::https_port') - $vhost_ssl_protocol = lookup('swh::deploy::webapp::vhost::ssl_protocol') - $vhost_ssl_honorcipherorder = lookup('swh::deploy::webapp::vhost::ssl_honorcipherorder') - $vhost_ssl_cipher = lookup('swh::deploy::webapp::vhost::ssl_cipher') $production_db_dir = lookup('swh::deploy::webapp::production_db_dir') $production_db_file = lookup('swh::deploy::webapp::production_db') $locked_endpoints = lookup('swh::deploy::webapp::locked_endpoints', Array, 'unique') $endpoint_directories = $locked_endpoints.map |$endpoint| { { path => "^${endpoint}", provider => 'locationmatch', auth_type => 'Basic', auth_name => 'Software Heritage development', auth_user_file => $vhost_basic_auth_file, auth_require => 'valid-user', } } # Install the necessary deps ::profile::swh::deploy::install_web_deps { 'swh-web': services => ['gunicorn-swh-webapp'], backport_list => 'swh::deploy::webapp::backported_packages', swh_packages => ['python3-swh.web'], } include ::gunicorn file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0755', } file {$conf_log_dir: ensure => directory, owner => 'root', group => $group, mode => '0770', } file {"$conf_log_dir/swh-web.log": ensure => present, owner => $user, group => $group, mode => '0770', } file {$vhost_docroot: ensure => directory, owner => 'root', group => $group, mode => '0755', } file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', content => inline_template("<%= @full_webapp_config.to_yaml %>\n"), notify => Service['gunicorn-swh-webapp'], } $storage_cfg = $full_webapp_config['storage'] if $storage_cfg['cls'] == 'cassandra' { include ::profile::swh::deploy::storage_cassandra } file {$production_db_dir: ensure => directory, owner => $user, group => $group, mode => '0755', } file {$production_db_file: ensure => present, owner => $user, group => $group, mode => '0664', } $sentry_dsn = lookup("swh::deploy::webapp::sentry_dsn", Optional[String], 'first', undef) $sentry_environment = lookup("swh::deploy::webapp::sentry_environment", Optional[String], 'first', undef) $sentry_swh_package = lookup("swh::deploy::webapp::sentry_swh_package", Optional[String], 'first', undef) ::gunicorn::instance {'swh-webapp': ensure => enabled, user => $user, group => $group, executable => 'django.core.wsgi:get_wsgi_application()', config_base_module => 'swh.web.gunicorn_config', settings => { bind => $backend_listen_address, workers => $backend_workers, worker_class => 'sync', timeout => $backend_http_timeout, graceful_timeout => $backend_reload_mercy, keepalive => $backend_http_keepalive, }, environment => { 'DJANGO_SETTINGS_MODULE' => 'swh.web.settings.production', 'SWH_SENTRY_DSN' => $sentry_dsn, 'SWH_SENTRY_ENVIRONMENT' => $sentry_environment, 'SWH_MAIN_PACKAGE' => $sentry_swh_package, }, } include ::profile::apache::common include ::apache::mod::proxy include ::apache::mod::headers ::apache::vhost {"${vhost_name}_non-ssl": servername => $vhost_name, serveraliases => $vhost_aliases, port => $vhost_port, docroot => $vhost_docroot, proxy_pass => [ { path => '/static', url => '!', }, { path => '/robots.txt', url => '!', }, { path => '/favicon.ico', url => '!', }, { path => '/', url => "http://${backend_listen_address}/", }, ], directories => [ { path => '/api', provider => 'location', allow => 'from all', satisfy => 'Any', headers => ['add Access-Control-Allow-Origin "*"'], }, { path => $static_dir, options => ['-Indexes'], }, ] + $endpoint_directories, aliases => [ { alias => '/static', path => $static_dir, }, { alias => '/robots.txt', path => "${static_dir}/robots.txt", }, ], # work around fix for CVE-2019-0220 introduced in Apache2 2.4.25-3+deb9u7 custom_fragment => 'MergeSlashes off', require => [ File[$vhost_basic_auth_file], ], } - include ::profile::hitch - realize(::Profile::Hitch::Ssl_cert[$cert_name]) - - include ::profile::varnish - ::profile::varnish::vhost {$vhost_name: - aliases => $vhost_aliases, - hsts_max_age => lookup('strict_transport_security::max_age'), - } - if $endpoint_directories { file {$vhost_basic_auth_file: ensure => present, owner => 'root', group => 'www-data', mode => '0640', content => $vhost_basic_auth_content, } } else { file {$vhost_basic_auth_file: ensure => absent, } } - $icinga_checks_file = lookup('icinga2::exported_checks::filename') - - @@::icinga2::object::service {"swh-webapp http redirect on ${::fqdn}": - service_name => 'swh webapp http redirect', - import => ['generic-service'], - host_name => $::fqdn, - check_command => 'http', - vars => { - http_address => $vhost_name, - http_vhost => $vhost_name, - http_port => $varnish_http_port, - http_uri => '/', - }, - target => $icinga_checks_file, - tag => 'icinga2::exported', - } - - @@::icinga2::object::service {"swh-webapp https on ${::fqdn}": - service_name => 'swh webapp', - import => ['generic-service'], - host_name => $::fqdn, - check_command => 'http', - vars => { - http_address => $vhost_name, - http_vhost => $vhost_name, - http_port => $vhost_ssl_port, - http_ssl => true, - http_sni => true, - http_uri => '/', - http_onredirect => sticky - }, - target => $icinga_checks_file, - tag => 'icinga2::exported', - } - - @@::icinga2::object::service {"swh-webapp https certificate ${::fqdn}": - service_name => 'swh webapp https certificate', - import => ['generic-service'], - host_name => $::fqdn, - check_command => 'http', - vars => { - http_address => $vhost_name, - http_vhost => $vhost_name, - http_port => $vhost_ssl_port, - http_ssl => true, - http_sni => true, - http_certificate => 15, - }, - target => $icinga_checks_file, - tag => 'icinga2::exported', - } - include ::profile::swh::deploy::webapp::icinga_checks profile::prometheus::export_scrape_config {"swh-webapp_${fqdn}": job => "swh-webapp", target => "${vhost_name}:443", scheme => "https", metrics_path => '/metrics/prometheus', labels => { vhost_name => $vhost_name, }, } include profile::filebeat profile::filebeat::log_input { 'webapp-non-ssl-access': paths => [ '/var/log/apache2/archive.softwareheritage.org_non-ssl_access.log' ], fields => { 'apache_log_type' => 'access_log' }, } }