diff --git a/site-modules/profile/manifests/swh/deploy/webapp.pp b/site-modules/profile/manifests/swh/deploy/webapp.pp index 051cf418..1dc4f677 100644 --- a/site-modules/profile/manifests/swh/deploy/webapp.pp +++ b/site-modules/profile/manifests/swh/deploy/webapp.pp @@ -1,266 +1,266 @@ # WebApp deployment class profile::swh::deploy::webapp { $conf_directory = lookup('swh::deploy::webapp::conf_directory') $conf_file = lookup('swh::deploy::webapp::conf_file') $user = lookup('swh::deploy::webapp::user') $group = lookup('swh::deploy::webapp::group') $webapp_config = lookup('swh::deploy::webapp::config') $conf_log_dir = lookup('swh::deploy::webapp::conf::log_dir') $webapp_settings_module = lookup('swh::deploy::webapp::django_settings_module') $backend_listen_host = lookup('swh::deploy::webapp::backend::listen::host') $backend_listen_port = lookup('swh::deploy::webapp::backend::listen::port') $backend_listen_address = "${backend_listen_host}:${backend_listen_port}" $backend_workers = lookup('swh::deploy::webapp::backend::workers') $backend_http_keepalive = lookup('swh::deploy::webapp::backend::http_keepalive') $backend_http_timeout = lookup('swh::deploy::webapp::backend::http_timeout') $backend_reload_mercy = lookup('swh::deploy::webapp::backend::reload_mercy') $static_dir = '/usr/share/swh/web/static' $cert_name = lookup('swh::deploy::webapp::vhost::letsencrypt_cert') $vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains'] $full_webapp_config = $webapp_config + {allowed_hosts => $vhosts} if $swh_hostname['fqdn'] in $vhosts { $vhost_name = $swh_hostname['fqdn'] } else { $vhost_name = $vhosts[0] } $vhost_aliases = delete($vhosts, $vhost_name) $vhost_access_log_format = lookup('swh::deploy::webapp::vhost::access_log_format') $vhost_port = lookup('apache::http_port') $vhost_docroot = "/var/www/${vhost_name}" $vhost_basic_auth_file = "${conf_directory}/http_auth" $vhost_basic_auth_content = lookup('swh::deploy::webapp::vhost::basic_auth_content', String, 'first', '') # Note that it's required by the ::profile::swh::deploy::webapp::icinga_checks $vhost_ssl_port = lookup('apache::https_port') $locked_endpoints = lookup('swh::deploy::webapp::locked_endpoints', Array, 'unique') $endpoint_directories = $locked_endpoints.map |$endpoint| { { path => "^${endpoint}", provider => 'locationmatch', auth_type => 'Basic', auth_name => 'Software Heritage development', auth_user_file => $vhost_basic_auth_file, auth_require => 'valid-user', } } $logfile = "${conf_log_dir}/swh-web.log" $pidfile = "/var/run/gunicorn/swh-webapp/pidfile" # Install the necessary deps ::profile::swh::deploy::install_web_deps { 'swh-web': services => ['gunicorn-swh-webapp'], backport_list => 'swh::deploy::webapp::backported_packages', swh_packages => ['python3-swh.web'], } include ::gunicorn file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0755', } file {$conf_log_dir: ensure => directory, owner => 'root', group => $group, mode => '0770', } file {$logfile: ensure => present, owner => $user, group => $group, mode => '0770', } # Template uses: # $logfile # $user # $group # $pidfile file {'/etc/logrotate.d/swh-webapp': ensure => file, owner => $user, group => $group, mode => '0644', content => template('profile/swh/logrotate-webapp.conf.erb'), require => File[$logfile], } file {$vhost_docroot: ensure => directory, owner => 'root', group => $group, mode => '0755', } file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', content => inline_template("<%= @full_webapp_config.to_yaml %>\n"), notify => Service['gunicorn-swh-webapp'], } $storage_cfg = $full_webapp_config['storage'] if $storage_cfg['cls'] == 'cassandra' { include ::profile::swh::deploy::storage_cassandra } $sentry_dsn = lookup('swh::deploy::webapp::sentry_dsn', Optional[String], 'first', undef) $sentry_environment = lookup('swh::deploy::webapp::sentry_environment', Optional[String], 'first', undef) $sentry_swh_package = lookup('swh::deploy::webapp::sentry_swh_package', Optional[String], 'first', undef) ::gunicorn::instance {'swh-webapp': ensure => enabled, user => $user, group => $group, executable => 'django.core.wsgi:get_wsgi_application()', config_base_module => 'swh.web.gunicorn_config', settings => { bind => $backend_listen_address, workers => $backend_workers, worker_class => 'sync', timeout => $backend_http_timeout, graceful_timeout => $backend_reload_mercy, keepalive => $backend_http_keepalive, }, environment => { 'DJANGO_SETTINGS_MODULE' => $webapp_settings_module, 'SWH_SENTRY_DSN' => $sentry_dsn, 'SWH_SENTRY_ENVIRONMENT' => $sentry_environment, 'SWH_MAIN_PACKAGE' => $sentry_swh_package, }, } include ::profile::apache::common include ::apache::mod::proxy include ::apache::mod::headers ::apache::vhost {"${vhost_name}_non-ssl": servername => $vhost_name, serveraliases => $vhost_aliases, port => $vhost_port, docroot => $vhost_docroot, proxy_pass => [ { path => '/static', url => '!', }, { path => '/robots.txt', url => '!', }, { path => '/favicon.ico', url => '!', }, { path => '/', url => "http://${backend_listen_address}/", }, ], directories => [ { path => $static_dir, options => ['-Indexes'], }, ] + $endpoint_directories, aliases => [ { alias => '/static', path => $static_dir, }, { alias => '/robots.txt', path => "${static_dir}/robots.txt", }, ], # work around fix for CVE-2019-0220 introduced in Apache2 2.4.25-3+deb9u7 custom_fragment => 'MergeSlashes off', require => [ File[$vhost_basic_auth_file], ], access_log_format => $vhost_access_log_format, } if $endpoint_directories { file {$vhost_basic_auth_file: ensure => present, owner => 'root', group => 'www-data', mode => '0640', content => $vhost_basic_auth_content, } } else { file {$vhost_basic_auth_file: ensure => absent, } } include ::profile::swh::deploy::webapp::icinga_checks profile::prometheus::export_scrape_config {"swh-webapp_${fqdn}": job => 'swh-webapp', target => "${vhost_name}:${vhost_ssl_port}", scheme => 'https', metrics_path => '/metrics/prometheus', labels => { vhost_name => $vhost_name, }, } include profile::filebeat # To remove when cleanup is done file {'/etc/filebeat/inputs.d/webapp-non-ssl-access.yml': ensure => absent, } profile::filebeat::log_input { "${vhost_name}-non-ssl-access": paths => [ "/var/log/apache2/${vhost_name}_non-ssl_access.log" ], fields => { 'apache_log_type' => 'access_log', 'environment' => $environment, 'vhost' => $vhost_name, 'application' => 'webapp', }, } # webapp update save code status routine $filename_refresh_status = 'refresh-savecodenow-statuses' # clean up old files $filepath_refresh_status = "/usr/local/bin/${filename_refresh_status}" file {$filepath_refresh_status: ensure => absent, owner => 'root', group => 'www-data', mode => '0755', } $activate_once_per_environment_webapp = lookup('swh::deploy::webapp::cron::refresh_statuses') # Template uses variables # - $user # - $group # - $webapp_settings_module # $update_savecodenow_service_name = "swh-webapp-update-savecodenow-statuses" $update_savecodenow_unit_template = "profile/swh/deploy/webapp/${update_savecodenow_service_name}.service.erb" $update_savecodenow_timer_name = "${update_savecodenow_service_name}.timer" $update_savecodenow_timer_template = "profile/swh/deploy/webapp/${update_savecodenow_timer_name}.erb" ::systemd::timer { $update_savecodenow_timer_name: timer_content => template($update_savecodenow_timer_template), service_content => template($update_savecodenow_unit_template), active => $activate_once_per_environment_webapp, enable => $activate_once_per_environment_webapp, - require => Package[$packages], + require => Profile::Swh::Deploy::Install_web_deps['swh-web'], } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_archive.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_archive.pp index 36b4a07e..12faaf08 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/loader_archive.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_archive.pp @@ -1,16 +1,13 @@ # Deployment for loader-archive class profile::swh::deploy::worker::loader_archive { include ::profile::swh::deploy::worker::base_loader_archive $private_tmp = lookup('swh::deploy::worker::loader_archive::private_tmp') ::profile::swh::deploy::worker::instance {'loader_archive': ensure => present, private_tmp => $private_tmp, sentry_name => 'loader_core', - require => [ - Package[$packages], - Package['lzip'], - ], + require => Class['profile::swh::deploy::worker::base_loader_archive'], } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_cran.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_cran.pp index 3419e902..986f87bf 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/loader_cran.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_cran.pp @@ -1,16 +1,13 @@ # Deployment for loader-cran class profile::swh::deploy::worker::loader_cran { include ::profile::swh::deploy::worker::base_loader_archive $private_tmp = lookup('swh::deploy::worker::loader_cran::private_tmp') ::profile::swh::deploy::worker::instance {'loader_cran': ensure => present, private_tmp => $private_tmp, sentry_name => 'loader_core', - require => [ - Package[$packages], - Package['lzip'], - ], + require => Class['profile::swh::deploy::worker::base_loader_archive'], } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp index 11c50e8e..45e8233d 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp @@ -1,20 +1,20 @@ # Deployment for loader-debian class profile::swh::deploy::worker::loader_debian { include ::profile::swh::deploy::worker::loader_package package {'dpkg-dev': ensure => 'present', } $private_tmp = lookup('swh::deploy::worker::loader_debian::private_tmp') ::profile::swh::deploy::worker::instance {'loader_debian': ensure => present, sentry_name => 'loader_core', private_tmp => $private_tmp, require => [ - Package[$packages], + Class['profile::swh::deploy::worker::loader_package'], Package['dpkg-dev'], ], } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_opam.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_opam.pp index 9f8ccc51..3cc4ae9b 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/loader_opam.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_opam.pp @@ -1,60 +1,60 @@ # Deployment for opam loader class profile::swh::deploy::worker::loader_opam { include ::profile::swh::deploy::worker::loader_package $private_tmp = lookup('swh::deploy::worker::loader_opam::private_tmp') $user = lookup('swh::deploy::worker::loader_opam::user') $group = lookup('swh::deploy::worker::loader_opam::group') $packages = ['opam'] package {$packages: ensure => 'present', } ::profile::swh::deploy::worker::instance {'loader_opam': ensure => present, private_tmp => $private_tmp, sentry_name => 'loader_core', require => [ - Package[$::profile::swh::deploy::loader_package::packages], Package[$packages], + Class['profile::swh::deploy::worker::loader_package'], ], } $opam_instances = lookup('swh::deploy::worker::opam::instances') $template_path = "profile/swh/deploy/loader_opam" $opam_root = lookup('swh::deploy::worker::opam::root_directory') $opam_manage_shared_state = "opam-manage-shared-state" $opam_manage_state_script = "/usr/local/bin/${opam_manage_shared_state}.sh" file {$opam_manage_state_script: ensure => 'file', owner => $user, group => $group, mode => '0755', content => template("${template_path}/${opam_manage_shared_state}.sh.erb"), } each ( $opam_instances ) | $instance, $instance_url | { $opam_manage_service_name = "${$opam_manage_shared_state}-${instance}" $opam_manage_shared_state_timer_name = "${opam_manage_service_name}.timer" # Templates uses variables # - $user # - $group # - $opam_root # - $opam_manage_service_name # - $command ::systemd::timer { $opam_manage_shared_state_timer_name: timer_content => template("${template_path}/${opam_manage_shared_state}.timer.erb"), service_content => template("${template_path}/${opam_manage_shared_state}.service.erb"), enable => true, require => [ Package[$packages], File[$opam_manage_state_script], ], } } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp b/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp index 6a7aa92f..a1a53e2d 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp @@ -1,26 +1,28 @@ # Deployment of a vault cooker class profile::swh::deploy::worker::vault_cooker { include ::profile::swh::deploy::base_vault $instance_name = 'vault_cooker' $config = lookup("swh::deploy::worker::${instance_name}::config", Hash, 'deep') if $config['graph'] { $extra_packages = [ "python3-swh.graph.client", ] package {$extra_packages: ensure => 'present', } + } else { + $extra_packages = [] } ::profile::swh::deploy::worker::instance {$instance_name: ensure => present, sentry_name => 'vault', send_task_events => true, require => [ Package[$extra_packages], - Package[$::profile::swh::deploy::base_vault::packages], + Class['profile::swh::deploy::base_vault'], ], } }