diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -682,6 +682,10 @@ varnish::storage_file: /var/lib/varnish/varnish_storage.bin # varnish::secret in private-data +swh::deploy::reverse_proxy::services: + - deposit + - webapp + letsencrypt::account_email: sysop+letsencrypt@softwareheritage.org letsencrypt::server: https://acme-v02.api.letsencrypt.org/directory diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -215,3 +215,14 @@ client_id: "swh.storage.journal_writer.%{::swh_hostname.short}" producer_config: message.max.bytes: 1000000000 + +## Reverse-proxy and frontend + +hitch::frontend: "[*]:443" +hitch::proxy_support: true + +varnish::http_port: 80 + +apache::http_port: 9080 +# Disable default vhost on port 80 +apache::default_vhost: false diff --git a/data/hostname/deposit.internal.staging.swh.network.yaml b/data/hostname/deposit.internal.staging.swh.network.yaml --- a/data/hostname/deposit.internal.staging.swh.network.yaml +++ b/data/hostname/deposit.internal.staging.swh.network.yaml @@ -4,17 +4,6 @@ netmask: 255.255.255.0 gateway: 192.168.130.1 -## frontend - -hitch::frontend: "[*]:443" -hitch::proxy_support: true - -varnish::http_port: 80 -apache::http_port: 9080 - -# Disable default vhost on port 80 -apache::default_vhost: false - ## deposit swh::deploy::deposit::media_root_directory: /srv/softwareheritage/deposit diff --git a/data/hostname/webapp.internal.staging.swh.network.yaml b/data/hostname/webapp.internal.staging.swh.network.yaml --- a/data/hostname/webapp.internal.staging.swh.network.yaml +++ b/data/hostname/webapp.internal.staging.swh.network.yaml @@ -4,15 +4,6 @@ netmask: 255.255.255.0 gateway: 192.168.130.1 -hitch::frontend: "[*]:443" -hitch::proxy_support: true - -varnish::http_port: 80 -apache::http_port: 9080 - -# Disable default vhost on port 80 -apache::default_vhost: false - swh::deploy::webapp::backend::workers: 16 swh::deploy::webapp::backend::http_keepalive: 5 swh::deploy::webapp::backend::http_timeout: 3600 diff --git a/data/hostname/webapp0.softwareheritage.org.yaml b/data/hostname/webapp0.softwareheritage.org.yaml --- a/data/hostname/webapp0.softwareheritage.org.yaml +++ b/data/hostname/webapp0.softwareheritage.org.yaml @@ -9,3 +9,6 @@ swh::remote_service::search::config: "%{alias('swh::remote_service::search::config::storage0')}" swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::azure')}" + +swh::deploy::reverse_proxy::services: + - webapp diff --git a/manifests/site.pp b/manifests/site.pp --- a/manifests/site.pp +++ b/manifests/site.pp @@ -17,11 +17,11 @@ } node 'moma.softwareheritage.org' { - include role::swh_api + include role::swh_rp_webapps } node 'webapp0.softwareheritage.org' { - include role::swh_base_api + include role::swh_rp_webapp } node 'saatchi.internal.softwareheritage.org' { @@ -160,8 +160,7 @@ } node 'webapp.internal.staging.swh.network' { - include role::swh_base_api - include profile::network + include role::swh_webapp } node 'deposit.internal.staging.swh.network' { @@ -172,6 +171,10 @@ include role::swh_vault } +node /^rp\d\.internal\.staging\.swh\.network$/ { + include role::swh_reverse_proxy +} + node 'journal0.internal.staging.swh.network' { include role::swh_journal_allinone } diff --git a/site-modules/profile/manifests/swh/deploy/deposit.pp b/site-modules/profile/manifests/swh/deploy/deposit.pp --- a/site-modules/profile/manifests/swh/deploy/deposit.pp +++ b/site-modules/profile/manifests/swh/deploy/deposit.pp @@ -162,19 +162,6 @@ ] } - include ::profile::hitch - realize(::Profile::Hitch::Ssl_cert[$cert_name]) - - include ::profile::varnish - $url_scheme = split($vhost_url, ':')[0] - - if $url_scheme == 'https' { - ::profile::varnish::vhost {$vhost_name: - aliases => $vhost_aliases, - hsts_max_age => lookup('strict_transport_security::max_age'), - } - } - file {$vhost_basic_auth_file: ensure => present, owner => 'root', @@ -217,55 +204,6 @@ } } - @@::icinga2::object::service {"swh-deposit http redirect on ${::fqdn}": - service_name => 'swh deposit http redirect', - import => ['generic-service'], - host_name => $::fqdn, - check_command => 'http', - vars => { - http_address => $vhost_name, - http_vhost => $vhost_name, - http_port => $vhost_port, - http_uri => '/', - }, - target => $icinga_checks_file, - tag => 'icinga2::exported', - } - - @@::icinga2::object::service {"swh-deposit https on ${::fqdn}": - service_name => 'swh deposit', - import => ['generic-service'], - host_name => $::fqdn, - check_command => 'http', - vars => { - http_address => $vhost_name, - http_vhost => $vhost_name, - http_port => $vhost_ssl_port, - http_ssl => true, - http_sni => true, - http_uri => '/', - http_onredirect => sticky - }, - target => $icinga_checks_file, - tag => 'icinga2::exported', - } - - @@::icinga2::object::service {"swh-deposit https certificate ${::fqdn}": - service_name => 'swh deposit https certificate', - import => ['generic-service'], - host_name => $::fqdn, - check_command => 'http', - vars => { - http_address => $vhost_name, - http_vhost => $vhost_name, - http_port => $vhost_ssl_port, - http_ssl => true, - http_sni => true, - http_certificate => 15, - }, - target => $icinga_checks_file, - tag => 'icinga2::exported', - } include profile::filebeat profile::filebeat::log_input { 'deposit-non-ssl-access': diff --git a/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp b/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp @@ -0,0 +1,89 @@ +# Reverse proxy to expose staging services +# https://forge.softwareheritage.org/T2747 +class profile::swh::deploy::reverse_proxy { + include ::profile::hitch + include ::profile::varnish + + $service_names = lookup('swh::deploy::reverse_proxy::services') + $varnish_http_port = lookup('varnish::http_port') + + each($service_names) |$service_name| { + # Retrieve certificate name + $cert_name = lookup("swh::deploy::${service_name}::vhost::letsencrypt_cert") + + # Retrieve the list of vhosts + $vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains'] + if $swh_hostname['fqdn'] in $vhosts { + $vhost_name = $swh_hostname['fqdn'] + } else { + $vhost_name = $vhosts[0] + } + # Compute aliases, removing the main vhost from the list + $vhost_aliases = delete($vhosts, $vhost_name) + + realize(::Profile::Hitch::Ssl_cert[$cert_name]) + ::profile::varnish::vhost {$vhost_name: + aliases => $vhost_aliases, + hsts_max_age => lookup('strict_transport_security::max_age'), + } + + $icinga_checks_file = lookup('icinga2::exported_checks::filename') + + # icinga alerts + @@::icinga2::object::service {"${service_name} http redirect on ${::fqdn}": + service_name => "swh ${service_name} http redirect", + import => ['generic-service'], + host_name => $::fqdn, + check_command => 'http', + vars => { + http_address => $vhost_name, + http_vhost => $vhost_name, + http_port => $varnish_http_port, + http_uri => '/', + }, + target => $icinga_checks_file, + tag => 'icinga2::exported', + } + + $vhost_ssl_port = lookup('apache::https_port') + + # $vhost_ssl_protocol = lookup('swh::deploy::webapp::vhost::ssl_protocol') + # $vhost_ssl_honorcipherorder = lookup('swh::deploy::webapp::vhost::ssl_honorcipherorder') + # $vhost_ssl_cipher = lookup('swh::deploy::webapp::vhost::ssl_cipher') + + @@::icinga2::object::service {"swh-${service_name} https on ${::fqdn}": + service_name => "swh ${service_name}", + import => ['generic-service'], + host_name => $::fqdn, + check_command => 'http', + vars => { + http_address => $vhost_name, + http_vhost => $vhost_name, + http_port => $vhost_ssl_port, + http_ssl => true, + http_sni => true, + http_uri => '/', + http_onredirect => sticky + }, + target => $icinga_checks_file, + tag => 'icinga2::exported', + } + + @@::icinga2::object::service {"swh-${service_name} https certificate ${::fqdn}": + service_name => "swh ${service_name} https certificate", + import => ['generic-service'], + host_name => $::fqdn, + check_command => 'http', + vars => { + http_address => $vhost_name, + http_vhost => $vhost_name, + http_port => $vhost_ssl_port, + http_ssl => true, + http_sni => true, + http_certificate => 15, + }, + target => $icinga_checks_file, + tag => 'icinga2::exported', + } + } +} diff --git a/site-modules/profile/manifests/swh/deploy/webapp.pp b/site-modules/profile/manifests/swh/deploy/webapp.pp --- a/site-modules/profile/manifests/swh/deploy/webapp.pp +++ b/site-modules/profile/manifests/swh/deploy/webapp.pp @@ -18,8 +18,6 @@ $static_dir = '/usr/share/swh/web/static' - $varnish_http_port = lookup('varnish::http_port') - $cert_name = lookup('swh::deploy::webapp::vhost::letsencrypt_cert') $vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains'] @@ -36,10 +34,6 @@ $vhost_docroot = "/var/www/${vhost_name}" $vhost_basic_auth_file = "${conf_directory}/http_auth" $vhost_basic_auth_content = lookup('swh::deploy::webapp::vhost::basic_auth_content', String, 'first', '') - $vhost_ssl_port = lookup('apache::https_port') - $vhost_ssl_protocol = lookup('swh::deploy::webapp::vhost::ssl_protocol') - $vhost_ssl_honorcipherorder = lookup('swh::deploy::webapp::vhost::ssl_honorcipherorder') - $vhost_ssl_cipher = lookup('swh::deploy::webapp::vhost::ssl_cipher') $production_db_dir = lookup('swh::deploy::webapp::production_db_dir') $production_db_file = lookup('swh::deploy::webapp::production_db') @@ -196,15 +190,6 @@ ], } - include ::profile::hitch - realize(::Profile::Hitch::Ssl_cert[$cert_name]) - - include ::profile::varnish - ::profile::varnish::vhost {$vhost_name: - aliases => $vhost_aliases, - hsts_max_age => lookup('strict_transport_security::max_age'), - } - if $endpoint_directories { file {$vhost_basic_auth_file: ensure => present, @@ -219,58 +204,6 @@ } } - $icinga_checks_file = lookup('icinga2::exported_checks::filename') - - @@::icinga2::object::service {"swh-webapp http redirect on ${::fqdn}": - service_name => 'swh webapp http redirect', - import => ['generic-service'], - host_name => $::fqdn, - check_command => 'http', - vars => { - http_address => $vhost_name, - http_vhost => $vhost_name, - http_port => $varnish_http_port, - http_uri => '/', - }, - target => $icinga_checks_file, - tag => 'icinga2::exported', - } - - @@::icinga2::object::service {"swh-webapp https on ${::fqdn}": - service_name => 'swh webapp', - import => ['generic-service'], - host_name => $::fqdn, - check_command => 'http', - vars => { - http_address => $vhost_name, - http_vhost => $vhost_name, - http_port => $vhost_ssl_port, - http_ssl => true, - http_sni => true, - http_uri => '/', - http_onredirect => sticky - }, - target => $icinga_checks_file, - tag => 'icinga2::exported', - } - - @@::icinga2::object::service {"swh-webapp https certificate ${::fqdn}": - service_name => 'swh webapp https certificate', - import => ['generic-service'], - host_name => $::fqdn, - check_command => 'http', - vars => { - http_address => $vhost_name, - http_vhost => $vhost_name, - http_port => $vhost_ssl_port, - http_ssl => true, - http_sni => true, - http_certificate => 15, - }, - target => $icinga_checks_file, - tag => 'icinga2::exported', - } - include ::profile::swh::deploy::webapp::icinga_checks profile::prometheus::export_scrape_config {"swh-webapp_${fqdn}": diff --git a/site-modules/role/manifests/swh_deposit.pp b/site-modules/role/manifests/swh_deposit.pp --- a/site-modules/role/manifests/swh_deposit.pp +++ b/site-modules/role/manifests/swh_deposit.pp @@ -1,7 +1,7 @@ +# Deposit api without reverse proxy class role::swh_deposit inherits role::swh_server { # Web UI include profile::swh::deploy::deposit - # Apache logs include profile::filebeat } diff --git a/site-modules/role/manifests/swh_reverse_proxy.pp b/site-modules/role/manifests/swh_reverse_proxy.pp new file mode 100644 --- /dev/null +++ b/site-modules/role/manifests/swh_reverse_proxy.pp @@ -0,0 +1,3 @@ +class role::swh_reverse_proxy inherits role::swh_server { + include profile::swh::deploy::reverse_proxy +} diff --git a/site-modules/role/manifests/swh_rp_webapp.pp b/site-modules/role/manifests/swh_rp_webapp.pp new file mode 100644 --- /dev/null +++ b/site-modules/role/manifests/swh_rp_webapp.pp @@ -0,0 +1,4 @@ +# Web ui profile with reverse proxy +class role::swh_rp_webapp inherits role::swh_webapp { + include profile::swh::deploy::reverse_proxy +} diff --git a/site-modules/role/manifests/swh_api.pp b/site-modules/role/manifests/swh_rp_webapps.pp rename from site-modules/role/manifests/swh_api.pp rename to site-modules/role/manifests/swh_rp_webapps.pp --- a/site-modules/role/manifests/swh_api.pp +++ b/site-modules/role/manifests/swh_rp_webapps.pp @@ -1,4 +1,4 @@ -class role::swh_api inherits role::swh_base_api { +class role::swh_rp_webapps inherits role::swh_rp_webapp { # Extra deposit and storage services include profile::swh::deploy::deposit include profile::swh::deploy::storage diff --git a/site-modules/role/manifests/swh_base_api.pp b/site-modules/role/manifests/swh_webapp.pp rename from site-modules/role/manifests/swh_base_api.pp rename to site-modules/role/manifests/swh_webapp.pp --- a/site-modules/role/manifests/swh_base_api.pp +++ b/site-modules/role/manifests/swh_webapp.pp @@ -1,8 +1,6 @@ -class role::swh_base_api inherits role::swh_server { - # Web UI +# Webapp api without reverse proxy +class role::swh_webapp inherits role::swh_server { include profile::memcached include profile::swh::deploy::webapp - - # Apache logs include profile::filebeat }