No OneTemporary
Actions

Size

39 KB

Subscribers

None

View Options

	diff --git a/site-modules/profile/manifests/swh/deploy/deposit.pp b/site-modules/profile/manifests/swh/deploy/deposit.pp
	index b31f6a43..8097d4fa 100644
	--- a/site-modules/profile/manifests/swh/deploy/deposit.pp
	+++ b/site-modules/profile/manifests/swh/deploy/deposit.pp
	@@ -1,240 +1,240 @@
	# Deployment of the swh.deposit server

	class profile::swh::deploy::deposit {
	$config_directory = lookup('swh::deploy::deposit::config_directory')
	$config_file = lookup('swh::deploy::deposit::config_file')
	$user = lookup('swh::deploy::deposit::user')
	$group = lookup('swh::deploy::deposit::group')
	$conf_hiera = lookup('swh::deploy::deposit::config')

	$static_dir = '/usr/lib/python3/dist-packages/swh/deposit/static'

	$backend_listen_host = lookup('swh::deploy::deposit::backend::listen::host')
	$backend_listen_port = lookup('swh::deploy::deposit::backend::listen::port')
	$backend_listen_address = "${backend_listen_host}:${backend_listen_port}"

	$backend_workers = lookup('swh::deploy::deposit::backend::workers')
	$backend_http_keepalive = lookup('swh::deploy::deposit::backend::http_keepalive')
	$backend_http_timeout = lookup('swh::deploy::deposit::backend::http_timeout')
	$backend_reload_mercy = lookup('swh::deploy::deposit::backend::reload_mercy')

	$vhost_url = lookup('swh::deploy::deposit::url')

	$cert_name = lookup('swh::deploy::deposit::vhost::letsencrypt_cert')
	$vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains']

	# authentication provider + optional keycloak config
	$conf_authent = lookup('swh::deploy::deposit::config::authentication')

	$full_conf = $conf_hiera + $conf_authent + {allowed_hosts => $vhosts}

	if $swh_hostname['fqdn'] in $vhosts {
	$vhost_name = $swh_hostname['fqdn']
	} else {
	$vhost_name = $vhosts[0]
	}
	$vhost_aliases = delete($vhosts, $vhost_name)

	$vhost_port = lookup('apache::http_port')
	$vhost_docroot = "/var/www/${vhost_name}"
	$vhost_basic_auth_file = "${config_directory}/http_auth"
	# swh::deploy::deposit::vhost::basic_auth_content in private
	$vhost_basic_auth_content = lookup('swh::deploy::deposit::vhost::basic_auth_content')
	$vhost_access_log_format = lookup('swh::deploy::deposit::vhost::access_log_format')
	$vhost_ssl_port = lookup('apache::https_port')
	$vhost_ssl_protocol = lookup('swh::deploy::deposit::vhost::ssl_protocol')
	$vhost_ssl_honorcipherorder = lookup('swh::deploy::deposit::vhost::ssl_honorcipherorder')
	$vhost_ssl_cipher = lookup('swh::deploy::deposit::vhost::ssl_cipher')
	$locked_endpoints = lookup('swh::deploy::deposit::locked_endpoints', Array, 'unique')

	$media_root_directory = lookup('swh::deploy::deposit::media_root_directory')

	include ::gunicorn

	# Install the necessary deps
	::profile::swh::deploy::install_web_deps { 'swh-deposit':
	+ ensure => present,
	services => ['gunicorn-swh-deposit'],
	backport_list => 'swh::deploy::deposit::backported_packages',
	swh_packages => ['python3-swh.deposit'],
	- ensure => present,
	}

	file {$config_directory:
	ensure => directory,
	owner => 'root',
	group => $group,
	mode => '0755',
	}

	# swh's configuration part (upload size, etc...)
	file {$config_file:
	ensure => present,
	owner => 'root',
	group => $group,
	mode => '0640',
	content => inline_template("<%= @full_conf.to_yaml %>\n"),
	notify => Service['gunicorn-swh-deposit'],
	}

	file {$media_root_directory:
	ensure => directory,
	owner => $user,
	group => $group,
	mode => '2750',
	}

	- $sentry_dsn = lookup("swh::deploy::deposit::sentry_dsn", Optional[String], 'first', undef)
	- $sentry_environment = lookup("swh::deploy::deposit::sentry_environment", Optional[String], 'first', undef)
	- $sentry_swh_package = lookup("swh::deploy::deposit::sentry_swh_package", Optional[String], 'first', undef)
	+ $sentry_dsn = lookup('swh::deploy::deposit::sentry_dsn', Optional[String], 'first', undef)
	+ $sentry_environment = lookup('swh::deploy::deposit::sentry_environment', Optional[String], 'first', undef)
	+ $sentry_swh_package = lookup('swh::deploy::deposit::sentry_swh_package', Optional[String], 'first', undef)

	::gunicorn::instance {'swh-deposit':
	- ensure => enabled,
	- user => $user,
	- group => $group,
	- executable => 'django.core.wsgi:get_wsgi_application()',
	+ ensure => enabled,
	+ user => $user,
	+ group => $group,
	+ executable => 'django.core.wsgi:get_wsgi_application()',
	config_base_module => 'swh.deposit.gunicorn_config',
	- environment => {
	+ environment => {
	'SWH_CONFIG_FILENAME' => $config_file,
	'DJANGO_SETTINGS_MODULE' => 'swh.deposit.settings.production',
	'SWH_SENTRY_DSN' => $sentry_dsn,
	'SWH_SENTRY_ENVIRONMENT' => $sentry_environment,
	'SWH_MAIN_PACKAGE' => $sentry_swh_package,
	},
	- settings => {
	+ settings => {
	bind => $backend_listen_address,
	workers => $backend_workers,
	worker_class => 'sync',
	timeout => $backend_http_timeout,
	graceful_timeout => $backend_reload_mercy,
	keepalive => $backend_http_keepalive,
	}
	}

	$endpoint_directories = $locked_endpoints.map \|$endpoint\| {
	{ path => "^${endpoint}",
	provider => 'locationmatch',
	auth_type => 'Basic',
	auth_name => 'Software Heritage Deposit',
	auth_user_file => $vhost_basic_auth_file,
	auth_require => 'valid-user',
	}
	}

	include ::profile::apache::common
	include ::apache::mod::proxy
	include ::apache::mod::headers

	::apache::vhost {"${vhost_name}_non-ssl":
	- servername => $vhost_name,
	- serveraliases => $vhost_aliases,
	- port => $vhost_port,
	- docroot => $vhost_docroot,
	- proxy_pass => [
	+ servername => $vhost_name,
	+ serveraliases => $vhost_aliases,
	+ port => $vhost_port,
	+ docroot => $vhost_docroot,
	+ proxy_pass => [
	{ path => '/static',
	url => '!',
	},
	{ path => '/robots.txt',
	url => '!',
	},
	{ path => '/favicon.ico',
	url => '!',
	},
	{ path => '/',
	url => "http://${backend_listen_address}/",
	},
	],
	- directories => [
	+ directories => [
	{ path => '/1',
	provider => 'location',
	allow => 'from all',
	satisfy => 'Any',
	headers => ['add Access-Control-Allow-Origin "*"'],
	},
	{ path => $static_dir,
	options => ['-Indexes'],
	},
	] + $endpoint_directories,
	- aliases => [
	+ aliases => [
	{ alias => '/static',
	path => $static_dir,
	},
	{ alias => '/robots.txt',
	path => "${static_dir}/robots.txt",
	},
	],
	access_log_format => $vhost_access_log_format,
	- require => [
	+ require => [
	File[$vhost_basic_auth_file],
	]
	}

	file {$vhost_basic_auth_file:
	ensure => present,
	owner => 'root',
	group => 'www-data',
	mode => '0640',
	content => $vhost_basic_auth_content,
	}

	$icinga_checks_file = lookup('icinga2::exported_checks::filename')

	@@::icinga2::object::service {"swh-deposit api (localhost on ${::fqdn})":
	service_name => 'swh-deposit api (localhost)',
	import => ['generic-service'],
	host_name => $::fqdn,
	check_command => 'http',
	command_endpoint => $::fqdn,
	vars => {
	http_address => '127.0.0.1',
	http_port => $backend_listen_port,
	http_uri => '/',
	http_string => 'The Software Heritage Deposit',
	},
	target => $icinga_checks_file,
	tag => 'icinga2::exported',
	}

	if $backend_listen_host != '127.0.0.1' {
	@@::icinga2::object::service {"swh-deposit api (remote on ${::fqdn})":
	service_name => 'swh-deposit api (remote)',
	import => ['generic-service'],
	host_name => $::fqdn,
	check_command => 'http',
	vars => {
	http_port => $backend_listen_port,
	http_uri => '/',
	http_string => 'The Software Heritage Deposit',
	},
	target => $icinga_checks_file,
	tag => 'icinga2::exported',
	}
	}

	# Install deposit end-to-end checks
	@@profile::icinga2::objects::e2e_checks_deposit {"End-to-end Deposit Test(s) in ${environment}":
	deposit_server => lookup('swh::deploy::deposit::e2e::server'),
	deposit_user => lookup('swh::deploy::deposit::e2e::user'),
	deposit_pass => lookup('swh::deploy::deposit::e2e::password'),
	deposit_collection => lookup('swh::deploy::deposit::e2e::collection'),
	deposit_poll_interval => lookup('swh::deploy::deposit::e2e::poll_interval'),
	deposit_archive => lookup('swh::deploy::deposit::e2e::archive'),
	deposit_metadata => lookup('swh::deploy::deposit::e2e::metadata'),
	environment => $environment,
	}

	include profile::filebeat
	# To remove when cleanup is done
	file {'/etc/filebeat/inputs.d/deposit-non-ssl-access.yml':
	ensure => absent,
	}
	profile::filebeat::log_input { "${vhost_name}-non-ssl-access":
	paths => [ "/var/log/apache2/${vhost_name}_non-ssl_access.log" ],
	fields => {
	'apache_log_type' => 'access_log',
	'environment' => $environment,
	'vhost' => $vhost_name,
	'application' => 'deposit',
	},
	}

	}
	diff --git a/site-modules/profile/manifests/swh/deploy/journal.pp b/site-modules/profile/manifests/swh/deploy/journal.pp
	index 294eb403..d694dae2 100644
	--- a/site-modules/profile/manifests/swh/deploy/journal.pp
	+++ b/site-modules/profile/manifests/swh/deploy/journal.pp
	@@ -1,38 +1,38 @@
	# Base Journal configuration

	class profile::swh::deploy::journal {
	$conf_directory = lookup('swh::deploy::journal::conf_directory')

	file {$conf_directory:
	ensure => 'directory',
	owner => 'swhworker',
	group => 'swhworker',
	mode => '0644',
	}

	$swh_packages = ['python3-swh.journal']

	$backported_packages = {
	'stretch' => ['librdkafka1'],
	}

	$pinned_packages = $backported_packages[$::lsbdistcodename]

	if $pinned_packages {
	::apt::pin {'swh-journal':
	explanation => 'Pin swh.journal dependencies to backports',
	codename => "${::lsbdistcodename}-backports",
	packages => $pinned_packages,
	priority => 990,
	}
	-> package {$swh_packages:
	- ensure => installed,
	+ ensure => installed,
	require => Apt::Source['softwareheritage'],
	}
	} else {
	package {$swh_packages:
	- ensure => installed,
	+ ensure => installed,
	require => Apt::Source['softwareheritage'],
	}
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp b/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp
	index 6242f4a9..eb4d6903 100644
	--- a/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp
	+++ b/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp
	@@ -1,99 +1,99 @@
	# Reverse proxy to expose staging services
	# https://forge.softwareheritage.org/T2747
	class profile::swh::deploy::reverse_proxy {
	include ::profile::hitch
	include ::profile::varnish

	$service_names = lookup('swh::deploy::reverse_proxy::services')
	$varnish_http_port = lookup('varnish::http_port')

	each($service_names) \|$service_name\| {
	# Retrieve certificate name
	$cert_name = lookup("swh::deploy::${service_name}::vhost::letsencrypt_cert")
	$backend_http_host = lookup("swh::deploy::${service_name}::reverse_proxy::backend_http_host")
	$backend_http_port = lookup("swh::deploy::${service_name}::reverse_proxy::backend_http_port")
	$websocket_support = lookup({
	- "name" => "swh::deploy::${service_name}::reverse_proxy::websocket_support",
	- "default_value" => false,
	+ 'name' => "swh::deploy::${service_name}::reverse_proxy::websocket_support",
	+ 'default_value' => false,
	})

	# Retrieve the list of vhosts
	$vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains']
	if $swh_hostname['fqdn'] in $vhosts {
	$vhost_name = $swh_hostname['fqdn']
	} else {
	$vhost_name = $vhosts[0]
	}
	# Compute aliases, removing the main vhost from the list
	$vhost_aliases = delete($vhosts, $vhost_name)

	realize(::Profile::Hitch::Ssl_cert[$cert_name])
	::profile::varnish::vhost {$vhost_name:
	aliases => $vhost_aliases,
	backend_name => $service_name,
	backend_http_host => $backend_http_host,
	backend_http_port => $backend_http_port,
	hsts_max_age => lookup('strict_transport_security::max_age'),
	websocket_support => $websocket_support,
	}

	$icinga_checks_file = lookup('icinga2::exported_checks::filename')

	# icinga alerts
	@@::icinga2::object::service {"${service_name} http redirect on ${::fqdn}":
	service_name => "swh ${service_name} http redirect",
	import => ['generic-service'],
	host_name => $::fqdn,
	check_command => 'http',
	vars => {
	http_address => $vhost_name,
	http_vhost => $vhost_name,
	http_port => $varnish_http_port,
	http_uri => '/',
	},
	target => $icinga_checks_file,
	tag => 'icinga2::exported',
	}

	$vhost_ssl_port = lookup('apache::https_port')

	# $vhost_ssl_protocol = lookup('swh::deploy::webapp::vhost::ssl_protocol')
	# $vhost_ssl_honorcipherorder = lookup('swh::deploy::webapp::vhost::ssl_honorcipherorder')
	# $vhost_ssl_cipher = lookup('swh::deploy::webapp::vhost::ssl_cipher')

	@@::icinga2::object::service {"swh-${service_name} https on ${::fqdn}":
	service_name => "swh ${service_name}",
	import => ['generic-service'],
	host_name => $::fqdn,
	check_command => 'http',
	vars => {
	http_address => $vhost_name,
	http_vhost => $vhost_name,
	http_port => $vhost_ssl_port,
	http_ssl => true,
	http_sni => true,
	http_uri => '/',
	http_onredirect => sticky
	},
	target => $icinga_checks_file,
	tag => 'icinga2::exported',
	}

	@@::icinga2::object::service {"swh-${service_name} https certificate ${::fqdn}":
	service_name => "swh ${service_name} https certificate",
	import => ['generic-service'],
	host_name => $::fqdn,
	check_command => 'http',
	vars => {
	http_address => $vhost_name,
	http_vhost => $vhost_name,
	http_port => $vhost_ssl_port,
	http_ssl => true,
	http_sni => true,
	http_certificate => 15,
	},
	target => $icinga_checks_file,
	tag => 'icinga2::exported',
	}
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/search/journal_client.pp b/site-modules/profile/manifests/swh/deploy/search/journal_client.pp
	index 30b6b122..321cbf11 100644
	--- a/site-modules/profile/manifests/swh/deploy/search/journal_client.pp
	+++ b/site-modules/profile/manifests/swh/deploy/search/journal_client.pp
	@@ -1,28 +1,28 @@
	# Deployment of the swh.search.journal_client
	class profile::swh::deploy::search::journal_client {
	include profile::swh::deploy::journal

	$service_types = lookup('swh::deploy::search::journal_client::service_types')

	$systemd_template_unit_name = 'swh-search-journal-client@.service'
	- $config_directory = lookup("swh::deploy::base_search::config_directory")
	+ $config_directory = lookup('swh::deploy::base_search::config_directory')

	$user = lookup('swh::deploy::base_search::user')
	$group = lookup('swh::deploy::base_search::group')

	# Uses:
	# - $config_directory
	# - $user
	# - $group
	systemd::unit_file {$systemd_template_unit_name:
	- ensure => 'present',
	+ ensure => 'present',
	content => template("profile/swh/deploy/journal/${systemd_template_unit_name}.erb"),
	}

	$service_types.each \| $service_type \| {
	profile::swh::deploy::search::journal_client_instance {$service_type:
	ensure => 'running',
	require => Package['python3-swh.search'],
	}
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/webapp.pp b/site-modules/profile/manifests/swh/deploy/webapp.pp
	index be6e2e56..35f0d154 100644
	--- a/site-modules/profile/manifests/swh/deploy/webapp.pp
	+++ b/site-modules/profile/manifests/swh/deploy/webapp.pp
	@@ -1,236 +1,236 @@
	# WebApp deployment
	class profile::swh::deploy::webapp {
	$conf_directory = lookup('swh::deploy::webapp::conf_directory')
	$conf_file = lookup('swh::deploy::webapp::conf_file')
	$user = lookup('swh::deploy::webapp::user')
	$group = lookup('swh::deploy::webapp::group')

	$webapp_config = lookup('swh::deploy::webapp::config')
	$conf_log_dir = lookup('swh::deploy::webapp::conf::log_dir')

	$webapp_settings_module = lookup('swh::deploy::webapp::django_settings_module')
	$backend_listen_host = lookup('swh::deploy::webapp::backend::listen::host')
	$backend_listen_port = lookup('swh::deploy::webapp::backend::listen::port')
	$backend_listen_address = "${backend_listen_host}:${backend_listen_port}"
	$backend_workers = lookup('swh::deploy::webapp::backend::workers')
	$backend_http_keepalive = lookup('swh::deploy::webapp::backend::http_keepalive')
	$backend_http_timeout = lookup('swh::deploy::webapp::backend::http_timeout')
	$backend_reload_mercy = lookup('swh::deploy::webapp::backend::reload_mercy')

	$static_dir = '/usr/share/swh/web/static'

	$cert_name = lookup('swh::deploy::webapp::vhost::letsencrypt_cert')
	$vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains']

	$full_webapp_config = $webapp_config + {allowed_hosts => $vhosts}

	if $swh_hostname['fqdn'] in $vhosts {
	$vhost_name = $swh_hostname['fqdn']
	} else {
	$vhost_name = $vhosts[0]
	}
	$vhost_aliases = delete($vhosts, $vhost_name)

	$vhost_access_log_format = lookup('swh::deploy::webapp::vhost::access_log_format')
	$vhost_port = lookup('apache::http_port')
	$vhost_docroot = "/var/www/${vhost_name}"
	$vhost_basic_auth_file = "${conf_directory}/http_auth"
	$vhost_basic_auth_content = lookup('swh::deploy::webapp::vhost::basic_auth_content', String, 'first', '')

	# Note that it's required by the ::profile::swh::deploy::webapp::icinga_checks
	$vhost_ssl_port = lookup('apache::https_port')

	$locked_endpoints = lookup('swh::deploy::webapp::locked_endpoints', Array, 'unique')

	$endpoint_directories = $locked_endpoints.map \|$endpoint\| {
	{ path => "^${endpoint}",
	provider => 'locationmatch',
	auth_type => 'Basic',
	auth_name => 'Software Heritage development',
	auth_user_file => $vhost_basic_auth_file,
	auth_require => 'valid-user',
	}
	}

	# Install the necessary deps
	::profile::swh::deploy::install_web_deps { 'swh-web':
	services => ['gunicorn-swh-webapp'],
	backport_list => 'swh::deploy::webapp::backported_packages',
	swh_packages => ['python3-swh.web'],
	}

	include ::gunicorn

	file {$conf_directory:
	ensure => directory,
	owner => 'root',
	group => $group,
	mode => '0755',
	}

	file {$conf_log_dir:
	ensure => directory,
	owner => 'root',
	group => $group,
	mode => '0770',
	}

	file {"${conf_log_dir}/swh-web.log":
	ensure => present,
	owner => $user,
	group => $group,
	mode => '0770',
	}

	file {$vhost_docroot:
	ensure => directory,
	owner => 'root',
	group => $group,
	mode => '0755',
	}

	file {$conf_file:
	ensure => present,
	owner => 'root',
	group => $group,
	mode => '0640',
	content => inline_template("<%= @full_webapp_config.to_yaml %>\n"),
	notify => Service['gunicorn-swh-webapp'],
	}

	$storage_cfg = $full_webapp_config['storage']
	if $storage_cfg['cls'] == 'cassandra' {
	include ::profile::swh::deploy::storage_cassandra
	}

	$sentry_dsn = lookup('swh::deploy::webapp::sentry_dsn', Optional[String], 'first', undef)
	$sentry_environment = lookup('swh::deploy::webapp::sentry_environment', Optional[String], 'first', undef)
	$sentry_swh_package = lookup('swh::deploy::webapp::sentry_swh_package', Optional[String], 'first', undef)

	::gunicorn::instance {'swh-webapp':
	ensure => enabled,
	user => $user,
	group => $group,
	executable => 'django.core.wsgi:get_wsgi_application()',
	config_base_module => 'swh.web.gunicorn_config',
	settings => {
	bind => $backend_listen_address,
	workers => $backend_workers,
	worker_class => 'sync',
	timeout => $backend_http_timeout,
	graceful_timeout => $backend_reload_mercy,
	keepalive => $backend_http_keepalive,
	},
	environment => {
	'DJANGO_SETTINGS_MODULE' => $webapp_settings_module,
	'SWH_SENTRY_DSN' => $sentry_dsn,
	'SWH_SENTRY_ENVIRONMENT' => $sentry_environment,
	'SWH_MAIN_PACKAGE' => $sentry_swh_package,
	},
	}

	include ::profile::apache::common
	include ::apache::mod::proxy
	include ::apache::mod::headers

	::apache::vhost {"${vhost_name}_non-ssl":
	- servername => $vhost_name,
	- serveraliases => $vhost_aliases,
	- port => $vhost_port,
	- docroot => $vhost_docroot,
	- proxy_pass => [
	+ servername => $vhost_name,
	+ serveraliases => $vhost_aliases,
	+ port => $vhost_port,
	+ docroot => $vhost_docroot,
	+ proxy_pass => [
	{ path => '/static',
	url => '!',
	},
	{ path => '/robots.txt',
	url => '!',
	},
	{ path => '/favicon.ico',
	url => '!',
	},
	{ path => '/',
	url => "http://${backend_listen_address}/",
	},
	],
	- directories => [
	+ directories => [
	{ path => $static_dir,
	options => ['-Indexes'],
	},
	] + $endpoint_directories,
	- aliases => [
	+ aliases => [
	{ alias => '/static',
	path => $static_dir,
	},
	{ alias => '/robots.txt',
	path => "${static_dir}/robots.txt",
	},
	],
	# work around fix for CVE-2019-0220 introduced in Apache2 2.4.25-3+deb9u7
	- custom_fragment => 'MergeSlashes off',
	- require => [
	+ custom_fragment => 'MergeSlashes off',
	+ require => [
	File[$vhost_basic_auth_file],
	],
	access_log_format => $vhost_access_log_format,
	}

	if $endpoint_directories {
	file {$vhost_basic_auth_file:
	ensure => present,
	owner => 'root',
	group => 'www-data',
	mode => '0640',
	content => $vhost_basic_auth_content,
	}
	} else {
	file {$vhost_basic_auth_file:
	ensure => absent,
	}
	}

	include ::profile::swh::deploy::webapp::icinga_checks

	profile::prometheus::export_scrape_config {"swh-webapp_${fqdn}":
	job => 'swh-webapp',
	target => "${vhost_name}:${vhost_ssl_port}",
	scheme => 'https',
	metrics_path => '/metrics/prometheus',
	labels => {
	vhost_name => $vhost_name,
	},
	}

	include profile::filebeat
	# To remove when cleanup is done
	file {'/etc/filebeat/inputs.d/webapp-non-ssl-access.yml':
	ensure => absent,
	}
	profile::filebeat::log_input { "${vhost_name}-non-ssl-access":
	paths => [ "/var/log/apache2/${vhost_name}_non-ssl_access.log" ],
	fields => {
	'apache_log_type' => 'access_log',
	'environment' => $environment,
	'vhost' => $vhost_name,
	'application' => 'webapp',
	},
	}

	- $filename_refresh_status = "refresh-savecodenow-statuses"
	+ $filename_refresh_status = 'refresh-savecodenow-statuses'
	$filepath_refresh_status = "/usr/local/bin/${filename_refresh_status}"
	file {$filepath_refresh_status:
	- ensure => present,
	- owner => 'root',
	- group => 'www-data',
	- mode => '0755',
	+ ensure => present,
	+ owner => 'root',
	+ group => 'www-data',
	+ mode => '0755',
	content => template("profile/swh/deploy/webapp/${filename_refresh_status}.sh.erb"),
	}

	$activate_once_per_environment_webapp = lookup('swh::deploy::webapp::cron::refresh_statuses')
	if $activate_once_per_environment_webapp {
	profile::cron::d {$filename_refresh_status:
	target => $filename_refresh_status,
	command => "chronic sh -c '${filepath_refresh_status}'",
	minute => '*',
	hour => '*',
	}
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/base.pp b/site-modules/profile/manifests/swh/deploy/worker/base.pp
	index dedfa3cc..c5361c5a 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/base.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/base.pp
	@@ -1,45 +1,45 @@
	# Base worker profile
	class profile::swh::deploy::worker::base {

	$systemd_template_unit_name = 'swh-worker@.service'
	$systemd_unit_name = 'swh-worker.service'
	$systemd_slice_name = 'system-swh\x2dworker.slice'

	$celery_hostname = $swh_hostname['internal_fqdn']

	package {'python3-swh.scheduler':
	ensure => installed,
	}

	::systemd::unit_file {$systemd_template_unit_name:
	- ensure => 'present',
	+ ensure => 'present',
	content => template("profile/swh/deploy/worker/${systemd_template_unit_name}.erb"),
	}

	::systemd::unit_file {$systemd_unit_name:
	ensure => 'present',
	source => "puppet:///modules/profile/swh/deploy/worker/${systemd_unit_name}",
	} ~> service {'swh-worker':
	ensure => running,
	enable => true,
	}

	::systemd::unit_file {$systemd_slice_name:
	ensure => 'present',
	source => "puppet:///modules/profile/swh/deploy/worker/${systemd_slice_name}",
	}

	profile::cron::d {'cleanup-workers-tmp':
	command => 'find /tmp -depth -mindepth 3 -maxdepth 3 -type d -ctime +2 -exec rm -rf {} \+',
	target => 'swh-worker',
	minute => 'fqdn_rand',
	hour => 'fqdn_rand/2',
	}

	file {'/usr/local/sbin/swh-worker-ping-restart':
	source => 'puppet:///modules/profile/swh/deploy/worker/swh-worker-ping-restart',
	owner => 'root',
	group => 'root',
	mode => '0755',
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/checker_deposit.pp b/site-modules/profile/manifests/swh/deploy/worker/checker_deposit.pp
	index 46b0927d..ce7cc95b 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/checker_deposit.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/checker_deposit.pp
	@@ -1,15 +1,15 @@
	# Deployment for deposit's archive checker
	class profile::swh::deploy::worker::checker_deposit {
	$packages = ['python3-swh.deposit.loader']

	package {$packages:
	ensure => 'present',
	}

	$private_tmp = lookup('swh::deploy::worker::checker_deposit::private_tmp')
	::profile::swh::deploy::worker::instance {'checker_deposit':
	- ensure => 'present',
	- sentry_name => 'deposit',
	- private_tmp => $private_tmp,
	+ ensure => 'present',
	+ sentry_name => 'deposit',
	+ private_tmp => $private_tmp,
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
	index e69727fb..c1b77e9e 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
	@@ -1,13 +1,13 @@
	# Deployment for swh-indexer-mimetype

	class profile::swh::deploy::worker::indexer_content_mimetype {
	include ::profile::swh::deploy::indexer

	Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_mimetype':
	- ensure => present,
	- sentry_name => 'indexer',
	- require => [
	+ ensure => present,
	+ sentry_name => 'indexer',
	+ require => [
	Class['profile::swh::deploy::indexer']
	],
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
	index 580492f8..3e4caf2e 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
	@@ -1,20 +1,20 @@
	# Deployment for swh-indexer-fossology-license

	class profile::swh::deploy::worker::indexer_fossology_license {
	include ::profile::swh::deploy::indexer
	$packages = ['fossology-nomossa']

	package {$packages:
	ensure => 'present',
	require => Apt::Source['softwareheritage'],
	}

	Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_fossology_license':
	- ensure => present,
	- sentry_name => 'indexer',
	- require => [
	+ ensure => present,
	+ sentry_name => 'indexer',
	+ require => [
	Class['profile::swh::deploy::indexer'],
	Package[$packages],
	],
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
	index ac201309..93e3004d 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
	@@ -1,13 +1,13 @@
	# Deployment for swh-indexer-origin-intrinsic-metadata

	class profile::swh::deploy::worker::indexer_origin_intrinsic_metadata {
	include ::profile::swh::deploy::indexer

	Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_origin_intrinsic_metadata':
	- ensure => present,
	- sentry_name => 'indexer',
	- require => [
	+ ensure => present,
	+ sentry_name => 'indexer',
	+ require => [
	Class['profile::swh::deploy::indexer'],
	],
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp
	index 65ef2866..2078d85e 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp
	@@ -1,13 +1,13 @@
	# Deployment for swh-indexer-rehash

	class profile::swh::deploy::worker::indexer_rehash {
	include ::profile::swh::deploy::indexer

	Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_rehash':
	- ensure => 'stopped',
	- sentry_name => 'indexer',
	- require => [
	+ ensure => 'stopped',
	+ sentry_name => 'indexer',
	+ require => [
	Class['profile::swh::deploy::indexer']
	],
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_archive.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_archive.pp
	index 8d8cdbf5..36b4a07e 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/loader_archive.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_archive.pp
	@@ -1,16 +1,16 @@
	# Deployment for loader-archive
	class profile::swh::deploy::worker::loader_archive {
	include ::profile::swh::deploy::worker::base_loader_archive

	$private_tmp = lookup('swh::deploy::worker::loader_archive::private_tmp')

	::profile::swh::deploy::worker::instance {'loader_archive':
	- ensure => present,
	- private_tmp => $private_tmp,
	- sentry_name => 'loader_core',
	- require => [
	+ ensure => present,
	+ private_tmp => $private_tmp,
	+ sentry_name => 'loader_core',
	+ require => [
	Package[$packages],
	Package['lzip'],
	],
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_cran.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_cran.pp
	index 7ab158be..3419e902 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/loader_cran.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_cran.pp
	@@ -1,16 +1,16 @@
	# Deployment for loader-cran
	class profile::swh::deploy::worker::loader_cran {
	include ::profile::swh::deploy::worker::base_loader_archive

	$private_tmp = lookup('swh::deploy::worker::loader_cran::private_tmp')

	::profile::swh::deploy::worker::instance {'loader_cran':
	- ensure => present,
	- private_tmp => $private_tmp,
	- sentry_name => 'loader_core',
	- require => [
	+ ensure => present,
	+ private_tmp => $private_tmp,
	+ sentry_name => 'loader_core',
	+ require => [
	Package[$packages],
	Package['lzip'],
	],
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp
	index b199fa47..11c50e8e 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp
	@@ -1,20 +1,20 @@
	# Deployment for loader-debian
	class profile::swh::deploy::worker::loader_debian {
	include ::profile::swh::deploy::worker::loader_package

	package {'dpkg-dev':
	ensure => 'present',
	}

	$private_tmp = lookup('swh::deploy::worker::loader_debian::private_tmp')

	::profile::swh::deploy::worker::instance {'loader_debian':
	- ensure => present,
	- sentry_name => 'loader_core',
	- private_tmp => $private_tmp,
	- require => [
	+ ensure => present,
	+ sentry_name => 'loader_core',
	+ private_tmp => $private_tmp,
	+ require => [
	Package[$packages],
	Package['dpkg-dev'],
	],
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp
	index 8dd1e74e..c528d060 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp
	@@ -1,11 +1,11 @@
	# Deployment for deposit's loader
	class profile::swh::deploy::worker::loader_deposit {
	include ::profile::swh::deploy::worker::loader_package

	$private_tmp = lookup('swh::deploy::worker::loader_deposit::private_tmp')
	::profile::swh::deploy::worker::instance {'loader_deposit':
	- ensure => 'present',
	- sentry_name => 'loader_core',
	- private_tmp => $private_tmp,
	+ ensure => 'present',
	+ sentry_name => 'loader_core',
	+ private_tmp => $private_tmp,
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp
	index 11d18101..a09155e6 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp
	@@ -1,11 +1,11 @@
	# Deployment for swh-loader-git (remote)
	class profile::swh::deploy::worker::loader_git {
	include ::profile::swh::deploy::base_loader_git

	::profile::swh::deploy::worker::instance {'loader_git':
	- ensure => present,
	- require => [
	+ ensure => present,
	+ require => [
	Package[$::profile::swh::deploy::base_loader_git::packages],
	],
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp
	index 1e9cfa7d..cca9f1d8 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp
	@@ -1,14 +1,14 @@
	# Deployment for high priority loader
	class profile::swh::deploy::worker::loader_high_priority {
	include ::profile::swh::deploy::base_loader_git
	include ::profile::swh::deploy::base_loader_mercurial
	include ::profile::swh::deploy::base_loader_svn

	$packages = $::profile::swh::deploy::base_loader_git::packages + $::profile::swh::deploy::base_loader_mercurial::packages + $::profile::swh::deploy::base_loader_svn::packages

	::profile::swh::deploy::worker::instance {'loader_high_priority':
	- ensure => present,
	- require => Package[$packages],
	+ ensure => present,
	+ require => Package[$packages],
	}

	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp
	index e956d742..101017a3 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp
	@@ -1,13 +1,13 @@
	# Deployment for swh-loader-mercurial (disk)
	class profile::swh::deploy::worker::loader_mercurial {
	include ::profile::swh::deploy::base_loader_mercurial
	$private_tmp = lookup('swh::deploy::worker::loader_mercurial::private_tmp')

	::profile::swh::deploy::worker::instance {'loader_mercurial':
	- ensure => 'present',
	- private_tmp => $private_tmp,
	- require => [
	+ ensure => 'present',
	+ private_tmp => $private_tmp,
	+ require => [
	Package[$::profile::swh::deploy::base_loader_mercurial::packages],
	],
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_nixguix.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_nixguix.pp
	index a84f157a..ffe837cb 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/loader_nixguix.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_nixguix.pp
	@@ -1,12 +1,12 @@
	# Deployment for loader-nixguix
	class profile::swh::deploy::worker::loader_nixguix {
	include ::profile::swh::deploy::worker::base_loader_archive

	$private_tmp = lookup('swh::deploy::worker::loader_nixguix::private_tmp')

	::profile::swh::deploy::worker::instance {'loader_nixguix':
	- ensure => present,
	- private_tmp => $private_tmp,
	- sentry_name => 'loader_core',
	+ ensure => present,
	+ private_tmp => $private_tmp,
	+ sentry_name => 'loader_core',
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_npm.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_npm.pp
	index a55a9967..fbec23a2 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/loader_npm.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_npm.pp
	@@ -1,19 +1,19 @@
	# Deployment for swh-loader-npm
	class profile::swh::deploy::worker::loader_npm {
	$private_tmp = lookup('swh::deploy::worker::loader_npm::private_tmp')

	$packages = ['python3-swh.loader.npm']

	package {$packages:
	ensure => 'present',
	}

	::profile::swh::deploy::worker::instance {'loader_npm':
	- ensure => present,
	- private_tmp => $private_tmp,
	- sentry_name => 'loader_core',
	- require => [
	+ ensure => present,
	+ private_tmp => $private_tmp,
	+ sentry_name => 'loader_core',
	+ require => [
	Package[$packages],
	],
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp
	index 445e7cc3..892bf85c 100644
	--- a/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp
	@@ -1,19 +1,19 @@
	# Deployment for swh-loader-pypi
	class profile::swh::deploy::worker::loader_pypi {
	$private_tmp = lookup('swh::deploy::worker::loader_pypi::private_tmp')

	$packages = ['python3-swh.loader.pypi']

	package {$packages:
	ensure => 'present',
	}

	::profile::swh::deploy::worker::instance {'loader_pypi':
	- ensure => present,
	- private_tmp => $private_tmp,
	- sentry_name => 'loader_core',
	- require => [
	+ ensure => present,
	+ private_tmp => $private_tmp,
	+ sentry_name => 'loader_core',
	+ require => [
	Package[$packages],
	],
	}
	}

File Metadata

Mime Type: text/x-diff
Expires: Sat, Jun 21, 7:00 PM (2 w, 6 d ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3268350

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions