diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml index ab58fb2c..aef6eda6 100644 --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -1,228 +1,233 @@ --- swh::deploy::environment: staging swh::deploy::worker::loader_nixguix::loglevel: debug swh::deploy::storage::db::host: db1.internal.staging.swh.network swh::deploy::storage::db::user: swh swh::deploy::storage::db::dbname: swh swh::deploy::indexer::storage::db::host: db1.internal.staging.swh.network swh::deploy::indexer::storage::db::user: swh-indexer swh::deploy::indexer::storage::db::dbname: swh-indexer swh::deploy::scheduler::db::host: db1.internal.staging.swh.network swh::deploy::scheduler::db::dbname: swh-scheduler swh::deploy::scheduler::db::user: swh-scheduler swh::deploy::deposit::db::host: db1.internal.staging.swh.network swh::deploy::deposit::db::dbuser: swh-deposit swh::deploy::deposit::db::dbname: swh-deposit swh::deploy::vault::db::host: db1.internal.staging.swh.network swh::deploy::vault::db::user: swh-vault swh::deploy::vault::db::dbname: swh-vault swh::deploy::worker::lister::db::host: db1.internal.staging.swh.network swh::deploy::worker::lister::db::user: swh-lister swh::deploy::worker::lister::db::name: swh-lister swh::deploy::worker::instances: - checker_deposit - loader_archive - loader_cran - loader_debian - loader_deposit - loader_nixguix - loader_git - loader_mercurial - loader_npm - loader_pypi - loader_svn - vault_cooker - lister - indexer_origin_intrinsic_metadata #### Rabbitmq instance to use # swh::deploy::worker::task_broker::password in private data swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@scheduler0.internal.staging.swh.network:5672/%2f" #### Storage/Indexer/Vault/Scheduler services to use in staging area swh::remote_service::storage::config::storage0: cls: remote args: url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/" swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::storage::config::writable: &swh_remote_service_storage_config_writable "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::vault::config::vault0: cls: remote args: url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/" swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::indexer::config::storage0: cls: remote url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/" swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::scheduler::config::scheduler0: cls: remote args: url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/" swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::deploy::deposit::url: http://deposit.internal.staging.swh.network # do not save pack swh::deploy::worker::loader_git::save_data_path: "" swh::deploy::worker::loader_git::concurrency: 1 zookeeper::clusters: rocquencourt: '1': journal0.internal.staging.swh.network kafka::clusters: rocquencourt: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - journal0.internal.staging.swh.network brokers: journal0.internal.staging.swh.network: id: 1 swh::deploy::journal::brokers: - journal0.internal.staging.swh.network swh::deploy::deposit::vhost::letsencrypt_cert: deposit_staging +swh::deploy::deposit::reverse_proxy::backend_http_host: deposit.internal.staging.swh.network +swh::deploy::deposit::reverse_proxy::backend_http_port: "%{alias('::profile::varnish::backend_http_port')}" + swh::deploy::webapp::vhost::letsencrypt_cert: archive_staging +swh::deploy::webapp::reverse_proxy::backend_http_host: webapp.internal.staging.swh.network +swh::deploy::webapp::reverse_proxy::backend_http_port: "%{alias('::profile::varnish::backend_http_port')}" swh::postgresql::version: '12' swh::postgresql::port: 5433 swh::postgresql::cluster_name: "%{lookup('swh::postgresql::version')}/main" swh::postgresql::datadir_base: "%{lookup('swh::base_directory')}/postgres" swh::postgresql::datadir: "%{lookup('swh::postgresql::datadir_base')}/%{lookup('swh::postgresql::cluster_name')}" swh::postgresql::listen_addresses: - 0.0.0.0 swh::postgresql::network_accesses: - 192.168.100.0/24 # Monitoring - 192.168.130.0/24 # Staging services swh::postgresql::shared_buffers: 32GB postgresql::server::config_entries: shared_buffers: "%{alias('swh::postgresql::shared_buffers')}" cluster_name: "%{alias('swh::postgresql::cluster_name')}" postgresql::globals::version: "%{alias('swh::postgresql::version')}" swh::dbs: storage: name: swh user: swh scheduler: name: swh-scheduler user: swh-scheduler vault: name: swh-vault user: swh-vault lister: name: swh-lister user: swh-lister deposit: name: swh-deposit user: swh-deposit indexer::storage: name: swh-indexer user: swh-indexer pgbouncer::auth_hba_file: "/etc/postgresql/%{lookup('swh::postgresql::cluster_name')}/pg_hba.conf" pgbouncer::common::listen_addresses: - 0.0.0.0 pgbouncer::databases: - source_db: swh host: localhost auth_user: postgres port: 5433 alias: staging-swh - source_db: swh-scheduler host: localhost auth_user: postgres port: 5433 alias: staging-swh-scheduler - source_db: swh-vault host: localhost auth_user: postgres port: 5433 alias: staging-swh-vault - source_db: swh-lister host: localhost auth_user: postgres port: 5433 alias: staging-swh-lister - source_db: swh-deposit host: localhost auth_user: postgres port: 5433 alias: staging-swh-deposit - source_db: swh-indexer host: localhost auth_user: postgres port: 5433 alias: staging-swh-indexer # open objstorage api swh::deploy::objstorage::backend::listen::host: 0.0.0.0 swh::deploy::objstorage::backend::workers: 4 swh::deploy::objstorage::directory: "%{hiera('swh::deploy::storage::directory')}" swh::deploy::objstorage::slicing: 0:1/1:5 swh::remote_service::objstorage::config: cls: pathslicing args: root: "%{hiera('swh::deploy::storage::directory')}" slicing: "%{hiera('swh::deploy::objstorage::slicing')}" # Deploy the storage server as a public resource swh::deploy::storage::backend::listen::host: 0.0.0.0 swh::deploy::storage::backend::workers: 4 swh::deploy::storage::backend::max_requests: 100 swh::deploy::storage::backend::max_requests_jitter: 10 # Deploy the indexer storage server as a public resource swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0 swh::deploy::indexer::storage::backend::workers: 4 nginx::worker_processes: 4 swh::deploy::storage::config: storage: cls: local args: db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" journal_writer: cls: kafka args: brokers: "%{alias('swh::deploy::journal::brokers')}" prefix: "%{alias('swh::deploy::journal::prefix')}" client_id: "swh.storage.journal_writer.%{::swh_hostname.short}" producer_config: message.max.bytes: 1000000000 ## Reverse-proxy and frontend hitch::frontend: "[*]:443" hitch::proxy_support: true varnish::http_port: 80 apache::http_port: 9080 # Disable default vhost on port 80 apache::default_vhost: false diff --git a/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp b/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp index 40be2883..10db678a 100644 --- a/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp +++ b/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp @@ -1,89 +1,93 @@ # Reverse proxy to expose staging services # https://forge.softwareheritage.org/T2747 class profile::swh::deploy::reverse_proxy { include ::profile::hitch include ::profile::varnish $service_names = lookup('swh::deploy::reverse_proxy::services') $varnish_http_port = lookup('varnish::http_port') each($service_names) |$service_name| { # Retrieve certificate name $cert_name = lookup("swh::deploy::${service_name}::vhost::letsencrypt_cert") + $backend_http_host = lookup("swh::deploy::${service_name}::reverse_proxy::backend_http_host") + $backend_http_port = lookup("swh::deploy::${service_name}::reverse_proxy::backend_http_port") # Retrieve the list of vhosts $vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains'] if $swh_hostname['fqdn'] in $vhosts { $vhost_name = $swh_hostname['fqdn'] } else { $vhost_name = $vhosts[0] } # Compute aliases, removing the main vhost from the list $vhost_aliases = delete($vhosts, $vhost_name) realize(::Profile::Hitch::Ssl_cert[$cert_name]) ::profile::varnish::vhost {$vhost_name: - aliases => $vhost_aliases, - hsts_max_age => lookup('strict_transport_security::max_age'), + aliases => $vhost_aliases, + backend_http_host => $backend_http_host, + backend_http_port => $backend_http_port, + hsts_max_age => lookup('strict_transport_security::max_age'), } $icinga_checks_file = lookup('icinga2::exported_checks::filename') # icinga alerts @@::icinga2::object::service {"${service_name} http redirect on ${::fqdn}": service_name => "swh ${service_name} http redirect", import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $vhost_name, http_vhost => $vhost_name, http_port => $varnish_http_port, http_uri => '/', }, target => $icinga_checks_file, tag => 'icinga2::exported', } $vhost_ssl_port = lookup('apache::https_port') # $vhost_ssl_protocol = lookup('swh::deploy::webapp::vhost::ssl_protocol') # $vhost_ssl_honorcipherorder = lookup('swh::deploy::webapp::vhost::ssl_honorcipherorder') # $vhost_ssl_cipher = lookup('swh::deploy::webapp::vhost::ssl_cipher') @@::icinga2::object::service {"swh-${service_name} https on ${::fqdn}": service_name => "swh ${service_name}", import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $vhost_name, http_vhost => $vhost_name, http_port => $vhost_ssl_port, http_ssl => true, http_sni => true, http_uri => '/', http_onredirect => sticky }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"swh-${service_name} https certificate ${::fqdn}": service_name => "swh ${service_name} https certificate", import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $vhost_name, http_vhost => $vhost_name, http_port => $vhost_ssl_port, http_ssl => true, http_sni => true, http_certificate => 15, }, target => $icinga_checks_file, tag => 'icinga2::exported', } } } diff --git a/site-modules/profile/manifests/varnish.pp b/site-modules/profile/manifests/varnish.pp index f3f82e8d..da2cb2f4 100644 --- a/site-modules/profile/manifests/varnish.pp +++ b/site-modules/profile/manifests/varnish.pp @@ -1,80 +1,83 @@ # Varnish configuration class profile::varnish { $includes_dir = '/etc/varnish/includes' $includes_vcl_name = 'includes.vcl' $includes_vcl = "/etc/varnish/${includes_vcl_name}" $http_port = lookup('varnish::http_port') $backend_http_port = lookup('varnish::backend_http_port') $listen = lookup('varnish::listen') $admin_listen = lookup('varnish::admin_listen') $admin_port = lookup('varnish::admin_port') $http2_support = lookup('varnish::http2_support') $secret = lookup('varnish::secret') $storage_type = lookup('varnish::storage_type') $storage_size = lookup('varnish::storage_size') $storage_file = lookup('varnish::storage_file') if $http2_support { $runtime_params = { feature => '+http2', } } else { $runtime_params = {} } if $::lsbdistcodename == 'stretch' { $extra_class_params = {} } else { $extra_class_params = { vcl_reload_cmd => '/usr/share/varnish/varnishreload', } } class {'::varnish': addrepo => false, listen => $listen, admin_listen => $admin_listen, admin_port => $admin_port, secret => $secret, storage_type => $storage_type, storage_size => $storage_size, storage_file => $storage_file, runtime_params => $runtime_params, * => $extra_class_params, } ::varnish::vcl {'/etc/varnish/default.vcl': content => template('profile/varnish/default.vcl.erb'), require => Concat[$includes_vcl], } file {$includes_dir: ensure => directory, owner => 'root', group => 'root', mode => '0644', require => Class['varnish::install'], notify => Exec['vcl_reload'], } concat {$includes_vcl: ensure => present, owner => 'root', group => 'root', mode => '0644', ensure_newline => true, require => Class['varnish::install'], notify => Exec['vcl_reload'], } concat::fragment {"${includes_vcl}:header": target => $includes_vcl, content => "# File managed with puppet (module profile::varnish)\n# All modifications will be lost\n\n", order => '00', } - include ::profile::varnish::default_vcls + ::profile::varnish::vcl_include {'synth_redirect': + order => '10', + content => file('profile/varnish/synth_redirect.vcl'), + } } diff --git a/site-modules/profile/manifests/varnish/default_vcls.pp b/site-modules/profile/manifests/varnish/default_vcls.pp deleted file mode 100644 index 9db8f9ce..00000000 --- a/site-modules/profile/manifests/varnish/default_vcls.pp +++ /dev/null @@ -1,13 +0,0 @@ -# Default VCLs included with the varnish profile - -class profile::varnish::default_vcls { - ::profile::varnish::vcl_include {'backend_default': - order => '01', - content => template('profile/varnish/backend_default.vcl.erb'), - } - - ::profile::varnish::vcl_include {'synth_redirect': - order => '10', - content => file('profile/varnish/synth_redirect.vcl'), - } -} diff --git a/site-modules/profile/manifests/varnish/vhost.pp b/site-modules/profile/manifests/varnish/vhost.pp index 6b9baef5..9f80348e 100644 --- a/site-modules/profile/manifests/varnish/vhost.pp +++ b/site-modules/profile/manifests/varnish/vhost.pp @@ -1,15 +1,23 @@ # Virtual host definition for varnish define profile::varnish::vhost ( String $servername = $title, String $order = '50', Array[String] $aliases = [], + String $backend_http_host, + String $backend_http_port, Optional[String] $vcl_recv_extra = undef, Optional[String] $vcl_deliver_extra = undef, Variant[Undef, String, Integer[1]] $hsts_max_age = undef, ) { + + ::profile::varnish::vcl_include {"backend_${servername}": + order => '01', + content => template('profile/varnish/backend.vcl.erb'), + } + ::profile::varnish::vcl_include {"vhost_${servername}": order => $order, content => template('profile/varnish/vhost.vcl.erb'), } } diff --git a/site-modules/profile/templates/varnish/backend.vcl.erb b/site-modules/profile/templates/varnish/backend.vcl.erb new file mode 100644 index 00000000..22b673db --- /dev/null +++ b/site-modules/profile/templates/varnish/backend.vcl.erb @@ -0,0 +1,11 @@ +# backend_default.vcl +# +# Default backend definition. +# +# File managed by puppet. All modifications will be lost. + +backend <%= Regexp.escape(@servername) %> +{ + .host = "<%= @backend_http_host %>"; + .port = "<%= @backend_http_port %>"; +} diff --git a/site-modules/profile/templates/varnish/backend_default.vcl.erb b/site-modules/profile/templates/varnish/backend_default.vcl.erb deleted file mode 100644 index 787d00df..00000000 --- a/site-modules/profile/templates/varnish/backend_default.vcl.erb +++ /dev/null @@ -1,10 +0,0 @@ -# backend_default.vcl -# -# Default backend definition. -# -# File managed by puppet. All modifications will be lost. - -backend default { - .host = "::1"; - .port = "<%= scope['::profile::varnish::backend_http_port'] %>"; -} diff --git a/site-modules/profile/templates/varnish/vhost.vcl.erb b/site-modules/profile/templates/varnish/vhost.vcl.erb index 0cb7b7af..dab14bb5 100644 --- a/site-modules/profile/templates/varnish/vhost.vcl.erb +++ b/site-modules/profile/templates/varnish/vhost.vcl.erb @@ -1,44 +1,45 @@ # vhost_<%= @servername %>.vcl # # Settings for the <%= @servername %> vhost # # File managed by puppet. All modifications will be lost. sub vcl_recv { if ( <% @aliases.each do |alias_| -%> req.http.host ~ "^(?i)<%= Regexp.escape(alias_) %>$" || <% end -%> req.http.host ~ "^(?i)<%= Regexp.escape(@servername) %>$" ) { if (std.port(server.ip) == <%= scope['::profile::varnish::http_port'] %>) { set req.http.x-redir = "https://" + req.http.host + req.url; return(synth(850, "Moved permanently")); } else { set req.http.X_FORWARDED_PROTO = "https"; + set req.backend_hint = <%= Regexp.escape(@servername) %>; } <% if @vcl_recv_extra -%> <%= @vcl_recv_extra %> <% end -%> } } <% if @hsts_max_age or @vcl_deliver_extra -%> sub vcl_deliver { if ( <% @aliases.each do |alias_| -%> req.http.host ~ "^(?i)<%= Regexp.escape(alias_) %>$" || <% end -%> req.http.host ~ "^(?i)<%= Regexp.escape(@servername) %>$" ) { <% if @hsts_max_age -%> if (std.port(server.ip) != <%= scope['::profile::varnish::http_port'] %>) { set resp.http.Strict-Transport-Security = "max-age=<%= @hsts_max_age %>;"; } <% end -%> <% if @vcl_deliver_extra -%> <%= @vcl_deliver_extra %> <% end -%> } } <% end -%>