Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9124353
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
39 KB
Subscribers
None
View Options
diff --git a/site-modules/profile/manifests/swh/deploy/deposit.pp b/site-modules/profile/manifests/swh/deploy/deposit.pp
index b31f6a43..8097d4fa 100644
--- a/site-modules/profile/manifests/swh/deploy/deposit.pp
+++ b/site-modules/profile/manifests/swh/deploy/deposit.pp
@@ -1,240 +1,240 @@
# Deployment of the swh.deposit server
class profile::swh::deploy::deposit {
$config_directory = lookup('swh::deploy::deposit::config_directory')
$config_file = lookup('swh::deploy::deposit::config_file')
$user = lookup('swh::deploy::deposit::user')
$group = lookup('swh::deploy::deposit::group')
$conf_hiera = lookup('swh::deploy::deposit::config')
$static_dir = '/usr/lib/python3/dist-packages/swh/deposit/static'
$backend_listen_host = lookup('swh::deploy::deposit::backend::listen::host')
$backend_listen_port = lookup('swh::deploy::deposit::backend::listen::port')
$backend_listen_address = "${backend_listen_host}:${backend_listen_port}"
$backend_workers = lookup('swh::deploy::deposit::backend::workers')
$backend_http_keepalive = lookup('swh::deploy::deposit::backend::http_keepalive')
$backend_http_timeout = lookup('swh::deploy::deposit::backend::http_timeout')
$backend_reload_mercy = lookup('swh::deploy::deposit::backend::reload_mercy')
$vhost_url = lookup('swh::deploy::deposit::url')
$cert_name = lookup('swh::deploy::deposit::vhost::letsencrypt_cert')
$vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains']
# authentication provider + optional keycloak config
$conf_authent = lookup('swh::deploy::deposit::config::authentication')
$full_conf = $conf_hiera + $conf_authent + {allowed_hosts => $vhosts}
if $swh_hostname['fqdn'] in $vhosts {
$vhost_name = $swh_hostname['fqdn']
} else {
$vhost_name = $vhosts[0]
}
$vhost_aliases = delete($vhosts, $vhost_name)
$vhost_port = lookup('apache::http_port')
$vhost_docroot = "/var/www/${vhost_name}"
$vhost_basic_auth_file = "${config_directory}/http_auth"
# swh::deploy::deposit::vhost::basic_auth_content in private
$vhost_basic_auth_content = lookup('swh::deploy::deposit::vhost::basic_auth_content')
$vhost_access_log_format = lookup('swh::deploy::deposit::vhost::access_log_format')
$vhost_ssl_port = lookup('apache::https_port')
$vhost_ssl_protocol = lookup('swh::deploy::deposit::vhost::ssl_protocol')
$vhost_ssl_honorcipherorder = lookup('swh::deploy::deposit::vhost::ssl_honorcipherorder')
$vhost_ssl_cipher = lookup('swh::deploy::deposit::vhost::ssl_cipher')
$locked_endpoints = lookup('swh::deploy::deposit::locked_endpoints', Array, 'unique')
$media_root_directory = lookup('swh::deploy::deposit::media_root_directory')
include ::gunicorn
# Install the necessary deps
::profile::swh::deploy::install_web_deps { 'swh-deposit':
+ ensure => present,
services => ['gunicorn-swh-deposit'],
backport_list => 'swh::deploy::deposit::backported_packages',
swh_packages => ['python3-swh.deposit'],
- ensure => present,
}
file {$config_directory:
ensure => directory,
owner => 'root',
group => $group,
mode => '0755',
}
# swh's configuration part (upload size, etc...)
file {$config_file:
ensure => present,
owner => 'root',
group => $group,
mode => '0640',
content => inline_template("<%= @full_conf.to_yaml %>\n"),
notify => Service['gunicorn-swh-deposit'],
}
file {$media_root_directory:
ensure => directory,
owner => $user,
group => $group,
mode => '2750',
}
- $sentry_dsn = lookup("swh::deploy::deposit::sentry_dsn", Optional[String], 'first', undef)
- $sentry_environment = lookup("swh::deploy::deposit::sentry_environment", Optional[String], 'first', undef)
- $sentry_swh_package = lookup("swh::deploy::deposit::sentry_swh_package", Optional[String], 'first', undef)
+ $sentry_dsn = lookup('swh::deploy::deposit::sentry_dsn', Optional[String], 'first', undef)
+ $sentry_environment = lookup('swh::deploy::deposit::sentry_environment', Optional[String], 'first', undef)
+ $sentry_swh_package = lookup('swh::deploy::deposit::sentry_swh_package', Optional[String], 'first', undef)
::gunicorn::instance {'swh-deposit':
- ensure => enabled,
- user => $user,
- group => $group,
- executable => 'django.core.wsgi:get_wsgi_application()',
+ ensure => enabled,
+ user => $user,
+ group => $group,
+ executable => 'django.core.wsgi:get_wsgi_application()',
config_base_module => 'swh.deposit.gunicorn_config',
- environment => {
+ environment => {
'SWH_CONFIG_FILENAME' => $config_file,
'DJANGO_SETTINGS_MODULE' => 'swh.deposit.settings.production',
'SWH_SENTRY_DSN' => $sentry_dsn,
'SWH_SENTRY_ENVIRONMENT' => $sentry_environment,
'SWH_MAIN_PACKAGE' => $sentry_swh_package,
},
- settings => {
+ settings => {
bind => $backend_listen_address,
workers => $backend_workers,
worker_class => 'sync',
timeout => $backend_http_timeout,
graceful_timeout => $backend_reload_mercy,
keepalive => $backend_http_keepalive,
}
}
$endpoint_directories = $locked_endpoints.map |$endpoint| {
{ path => "^${endpoint}",
provider => 'locationmatch',
auth_type => 'Basic',
auth_name => 'Software Heritage Deposit',
auth_user_file => $vhost_basic_auth_file,
auth_require => 'valid-user',
}
}
include ::profile::apache::common
include ::apache::mod::proxy
include ::apache::mod::headers
::apache::vhost {"${vhost_name}_non-ssl":
- servername => $vhost_name,
- serveraliases => $vhost_aliases,
- port => $vhost_port,
- docroot => $vhost_docroot,
- proxy_pass => [
+ servername => $vhost_name,
+ serveraliases => $vhost_aliases,
+ port => $vhost_port,
+ docroot => $vhost_docroot,
+ proxy_pass => [
{ path => '/static',
url => '!',
},
{ path => '/robots.txt',
url => '!',
},
{ path => '/favicon.ico',
url => '!',
},
{ path => '/',
url => "http://${backend_listen_address}/",
},
],
- directories => [
+ directories => [
{ path => '/1',
provider => 'location',
allow => 'from all',
satisfy => 'Any',
headers => ['add Access-Control-Allow-Origin "*"'],
},
{ path => $static_dir,
options => ['-Indexes'],
},
] + $endpoint_directories,
- aliases => [
+ aliases => [
{ alias => '/static',
path => $static_dir,
},
{ alias => '/robots.txt',
path => "${static_dir}/robots.txt",
},
],
access_log_format => $vhost_access_log_format,
- require => [
+ require => [
File[$vhost_basic_auth_file],
]
}
file {$vhost_basic_auth_file:
ensure => present,
owner => 'root',
group => 'www-data',
mode => '0640',
content => $vhost_basic_auth_content,
}
$icinga_checks_file = lookup('icinga2::exported_checks::filename')
@@::icinga2::object::service {"swh-deposit api (localhost on ${::fqdn})":
service_name => 'swh-deposit api (localhost)',
import => ['generic-service'],
host_name => $::fqdn,
check_command => 'http',
command_endpoint => $::fqdn,
vars => {
http_address => '127.0.0.1',
http_port => $backend_listen_port,
http_uri => '/',
http_string => 'The Software Heritage Deposit',
},
target => $icinga_checks_file,
tag => 'icinga2::exported',
}
if $backend_listen_host != '127.0.0.1' {
@@::icinga2::object::service {"swh-deposit api (remote on ${::fqdn})":
service_name => 'swh-deposit api (remote)',
import => ['generic-service'],
host_name => $::fqdn,
check_command => 'http',
vars => {
http_port => $backend_listen_port,
http_uri => '/',
http_string => 'The Software Heritage Deposit',
},
target => $icinga_checks_file,
tag => 'icinga2::exported',
}
}
# Install deposit end-to-end checks
@@profile::icinga2::objects::e2e_checks_deposit {"End-to-end Deposit Test(s) in ${environment}":
deposit_server => lookup('swh::deploy::deposit::e2e::server'),
deposit_user => lookup('swh::deploy::deposit::e2e::user'),
deposit_pass => lookup('swh::deploy::deposit::e2e::password'),
deposit_collection => lookup('swh::deploy::deposit::e2e::collection'),
deposit_poll_interval => lookup('swh::deploy::deposit::e2e::poll_interval'),
deposit_archive => lookup('swh::deploy::deposit::e2e::archive'),
deposit_metadata => lookup('swh::deploy::deposit::e2e::metadata'),
environment => $environment,
}
include profile::filebeat
# To remove when cleanup is done
file {'/etc/filebeat/inputs.d/deposit-non-ssl-access.yml':
ensure => absent,
}
profile::filebeat::log_input { "${vhost_name}-non-ssl-access":
paths => [ "/var/log/apache2/${vhost_name}_non-ssl_access.log" ],
fields => {
'apache_log_type' => 'access_log',
'environment' => $environment,
'vhost' => $vhost_name,
'application' => 'deposit',
},
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/journal.pp b/site-modules/profile/manifests/swh/deploy/journal.pp
index 294eb403..d694dae2 100644
--- a/site-modules/profile/manifests/swh/deploy/journal.pp
+++ b/site-modules/profile/manifests/swh/deploy/journal.pp
@@ -1,38 +1,38 @@
# Base Journal configuration
class profile::swh::deploy::journal {
$conf_directory = lookup('swh::deploy::journal::conf_directory')
file {$conf_directory:
ensure => 'directory',
owner => 'swhworker',
group => 'swhworker',
mode => '0644',
}
$swh_packages = ['python3-swh.journal']
$backported_packages = {
'stretch' => ['librdkafka1'],
}
$pinned_packages = $backported_packages[$::lsbdistcodename]
if $pinned_packages {
::apt::pin {'swh-journal':
explanation => 'Pin swh.journal dependencies to backports',
codename => "${::lsbdistcodename}-backports",
packages => $pinned_packages,
priority => 990,
}
-> package {$swh_packages:
- ensure => installed,
+ ensure => installed,
require => Apt::Source['softwareheritage'],
}
} else {
package {$swh_packages:
- ensure => installed,
+ ensure => installed,
require => Apt::Source['softwareheritage'],
}
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp b/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp
index 6242f4a9..eb4d6903 100644
--- a/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp
+++ b/site-modules/profile/manifests/swh/deploy/reverse_proxy.pp
@@ -1,99 +1,99 @@
# Reverse proxy to expose staging services
# https://forge.softwareheritage.org/T2747
class profile::swh::deploy::reverse_proxy {
include ::profile::hitch
include ::profile::varnish
$service_names = lookup('swh::deploy::reverse_proxy::services')
$varnish_http_port = lookup('varnish::http_port')
each($service_names) |$service_name| {
# Retrieve certificate name
$cert_name = lookup("swh::deploy::${service_name}::vhost::letsencrypt_cert")
$backend_http_host = lookup("swh::deploy::${service_name}::reverse_proxy::backend_http_host")
$backend_http_port = lookup("swh::deploy::${service_name}::reverse_proxy::backend_http_port")
$websocket_support = lookup({
- "name" => "swh::deploy::${service_name}::reverse_proxy::websocket_support",
- "default_value" => false,
+ 'name' => "swh::deploy::${service_name}::reverse_proxy::websocket_support",
+ 'default_value' => false,
})
# Retrieve the list of vhosts
$vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains']
if $swh_hostname['fqdn'] in $vhosts {
$vhost_name = $swh_hostname['fqdn']
} else {
$vhost_name = $vhosts[0]
}
# Compute aliases, removing the main vhost from the list
$vhost_aliases = delete($vhosts, $vhost_name)
realize(::Profile::Hitch::Ssl_cert[$cert_name])
::profile::varnish::vhost {$vhost_name:
aliases => $vhost_aliases,
backend_name => $service_name,
backend_http_host => $backend_http_host,
backend_http_port => $backend_http_port,
hsts_max_age => lookup('strict_transport_security::max_age'),
websocket_support => $websocket_support,
}
$icinga_checks_file = lookup('icinga2::exported_checks::filename')
# icinga alerts
@@::icinga2::object::service {"${service_name} http redirect on ${::fqdn}":
service_name => "swh ${service_name} http redirect",
import => ['generic-service'],
host_name => $::fqdn,
check_command => 'http',
vars => {
http_address => $vhost_name,
http_vhost => $vhost_name,
http_port => $varnish_http_port,
http_uri => '/',
},
target => $icinga_checks_file,
tag => 'icinga2::exported',
}
$vhost_ssl_port = lookup('apache::https_port')
# $vhost_ssl_protocol = lookup('swh::deploy::webapp::vhost::ssl_protocol')
# $vhost_ssl_honorcipherorder = lookup('swh::deploy::webapp::vhost::ssl_honorcipherorder')
# $vhost_ssl_cipher = lookup('swh::deploy::webapp::vhost::ssl_cipher')
@@::icinga2::object::service {"swh-${service_name} https on ${::fqdn}":
service_name => "swh ${service_name}",
import => ['generic-service'],
host_name => $::fqdn,
check_command => 'http',
vars => {
http_address => $vhost_name,
http_vhost => $vhost_name,
http_port => $vhost_ssl_port,
http_ssl => true,
http_sni => true,
http_uri => '/',
http_onredirect => sticky
},
target => $icinga_checks_file,
tag => 'icinga2::exported',
}
@@::icinga2::object::service {"swh-${service_name} https certificate ${::fqdn}":
service_name => "swh ${service_name} https certificate",
import => ['generic-service'],
host_name => $::fqdn,
check_command => 'http',
vars => {
http_address => $vhost_name,
http_vhost => $vhost_name,
http_port => $vhost_ssl_port,
http_ssl => true,
http_sni => true,
http_certificate => 15,
},
target => $icinga_checks_file,
tag => 'icinga2::exported',
}
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/search/journal_client.pp b/site-modules/profile/manifests/swh/deploy/search/journal_client.pp
index 30b6b122..321cbf11 100644
--- a/site-modules/profile/manifests/swh/deploy/search/journal_client.pp
+++ b/site-modules/profile/manifests/swh/deploy/search/journal_client.pp
@@ -1,28 +1,28 @@
# Deployment of the swh.search.journal_client
class profile::swh::deploy::search::journal_client {
include profile::swh::deploy::journal
$service_types = lookup('swh::deploy::search::journal_client::service_types')
$systemd_template_unit_name = 'swh-search-journal-client@.service'
- $config_directory = lookup("swh::deploy::base_search::config_directory")
+ $config_directory = lookup('swh::deploy::base_search::config_directory')
$user = lookup('swh::deploy::base_search::user')
$group = lookup('swh::deploy::base_search::group')
# Uses:
# - $config_directory
# - $user
# - $group
systemd::unit_file {$systemd_template_unit_name:
- ensure => 'present',
+ ensure => 'present',
content => template("profile/swh/deploy/journal/${systemd_template_unit_name}.erb"),
}
$service_types.each | $service_type | {
profile::swh::deploy::search::journal_client_instance {$service_type:
ensure => 'running',
require => Package['python3-swh.search'],
}
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/webapp.pp b/site-modules/profile/manifests/swh/deploy/webapp.pp
index be6e2e56..35f0d154 100644
--- a/site-modules/profile/manifests/swh/deploy/webapp.pp
+++ b/site-modules/profile/manifests/swh/deploy/webapp.pp
@@ -1,236 +1,236 @@
# WebApp deployment
class profile::swh::deploy::webapp {
$conf_directory = lookup('swh::deploy::webapp::conf_directory')
$conf_file = lookup('swh::deploy::webapp::conf_file')
$user = lookup('swh::deploy::webapp::user')
$group = lookup('swh::deploy::webapp::group')
$webapp_config = lookup('swh::deploy::webapp::config')
$conf_log_dir = lookup('swh::deploy::webapp::conf::log_dir')
$webapp_settings_module = lookup('swh::deploy::webapp::django_settings_module')
$backend_listen_host = lookup('swh::deploy::webapp::backend::listen::host')
$backend_listen_port = lookup('swh::deploy::webapp::backend::listen::port')
$backend_listen_address = "${backend_listen_host}:${backend_listen_port}"
$backend_workers = lookup('swh::deploy::webapp::backend::workers')
$backend_http_keepalive = lookup('swh::deploy::webapp::backend::http_keepalive')
$backend_http_timeout = lookup('swh::deploy::webapp::backend::http_timeout')
$backend_reload_mercy = lookup('swh::deploy::webapp::backend::reload_mercy')
$static_dir = '/usr/share/swh/web/static'
$cert_name = lookup('swh::deploy::webapp::vhost::letsencrypt_cert')
$vhosts = lookup('letsencrypt::certificates')[$cert_name]['domains']
$full_webapp_config = $webapp_config + {allowed_hosts => $vhosts}
if $swh_hostname['fqdn'] in $vhosts {
$vhost_name = $swh_hostname['fqdn']
} else {
$vhost_name = $vhosts[0]
}
$vhost_aliases = delete($vhosts, $vhost_name)
$vhost_access_log_format = lookup('swh::deploy::webapp::vhost::access_log_format')
$vhost_port = lookup('apache::http_port')
$vhost_docroot = "/var/www/${vhost_name}"
$vhost_basic_auth_file = "${conf_directory}/http_auth"
$vhost_basic_auth_content = lookup('swh::deploy::webapp::vhost::basic_auth_content', String, 'first', '')
# Note that it's required by the ::profile::swh::deploy::webapp::icinga_checks
$vhost_ssl_port = lookup('apache::https_port')
$locked_endpoints = lookup('swh::deploy::webapp::locked_endpoints', Array, 'unique')
$endpoint_directories = $locked_endpoints.map |$endpoint| {
{ path => "^${endpoint}",
provider => 'locationmatch',
auth_type => 'Basic',
auth_name => 'Software Heritage development',
auth_user_file => $vhost_basic_auth_file,
auth_require => 'valid-user',
}
}
# Install the necessary deps
::profile::swh::deploy::install_web_deps { 'swh-web':
services => ['gunicorn-swh-webapp'],
backport_list => 'swh::deploy::webapp::backported_packages',
swh_packages => ['python3-swh.web'],
}
include ::gunicorn
file {$conf_directory:
ensure => directory,
owner => 'root',
group => $group,
mode => '0755',
}
file {$conf_log_dir:
ensure => directory,
owner => 'root',
group => $group,
mode => '0770',
}
file {"${conf_log_dir}/swh-web.log":
ensure => present,
owner => $user,
group => $group,
mode => '0770',
}
file {$vhost_docroot:
ensure => directory,
owner => 'root',
group => $group,
mode => '0755',
}
file {$conf_file:
ensure => present,
owner => 'root',
group => $group,
mode => '0640',
content => inline_template("<%= @full_webapp_config.to_yaml %>\n"),
notify => Service['gunicorn-swh-webapp'],
}
$storage_cfg = $full_webapp_config['storage']
if $storage_cfg['cls'] == 'cassandra' {
include ::profile::swh::deploy::storage_cassandra
}
$sentry_dsn = lookup('swh::deploy::webapp::sentry_dsn', Optional[String], 'first', undef)
$sentry_environment = lookup('swh::deploy::webapp::sentry_environment', Optional[String], 'first', undef)
$sentry_swh_package = lookup('swh::deploy::webapp::sentry_swh_package', Optional[String], 'first', undef)
::gunicorn::instance {'swh-webapp':
ensure => enabled,
user => $user,
group => $group,
executable => 'django.core.wsgi:get_wsgi_application()',
config_base_module => 'swh.web.gunicorn_config',
settings => {
bind => $backend_listen_address,
workers => $backend_workers,
worker_class => 'sync',
timeout => $backend_http_timeout,
graceful_timeout => $backend_reload_mercy,
keepalive => $backend_http_keepalive,
},
environment => {
'DJANGO_SETTINGS_MODULE' => $webapp_settings_module,
'SWH_SENTRY_DSN' => $sentry_dsn,
'SWH_SENTRY_ENVIRONMENT' => $sentry_environment,
'SWH_MAIN_PACKAGE' => $sentry_swh_package,
},
}
include ::profile::apache::common
include ::apache::mod::proxy
include ::apache::mod::headers
::apache::vhost {"${vhost_name}_non-ssl":
- servername => $vhost_name,
- serveraliases => $vhost_aliases,
- port => $vhost_port,
- docroot => $vhost_docroot,
- proxy_pass => [
+ servername => $vhost_name,
+ serveraliases => $vhost_aliases,
+ port => $vhost_port,
+ docroot => $vhost_docroot,
+ proxy_pass => [
{ path => '/static',
url => '!',
},
{ path => '/robots.txt',
url => '!',
},
{ path => '/favicon.ico',
url => '!',
},
{ path => '/',
url => "http://${backend_listen_address}/",
},
],
- directories => [
+ directories => [
{ path => $static_dir,
options => ['-Indexes'],
},
] + $endpoint_directories,
- aliases => [
+ aliases => [
{ alias => '/static',
path => $static_dir,
},
{ alias => '/robots.txt',
path => "${static_dir}/robots.txt",
},
],
# work around fix for CVE-2019-0220 introduced in Apache2 2.4.25-3+deb9u7
- custom_fragment => 'MergeSlashes off',
- require => [
+ custom_fragment => 'MergeSlashes off',
+ require => [
File[$vhost_basic_auth_file],
],
access_log_format => $vhost_access_log_format,
}
if $endpoint_directories {
file {$vhost_basic_auth_file:
ensure => present,
owner => 'root',
group => 'www-data',
mode => '0640',
content => $vhost_basic_auth_content,
}
} else {
file {$vhost_basic_auth_file:
ensure => absent,
}
}
include ::profile::swh::deploy::webapp::icinga_checks
profile::prometheus::export_scrape_config {"swh-webapp_${fqdn}":
job => 'swh-webapp',
target => "${vhost_name}:${vhost_ssl_port}",
scheme => 'https',
metrics_path => '/metrics/prometheus',
labels => {
vhost_name => $vhost_name,
},
}
include profile::filebeat
# To remove when cleanup is done
file {'/etc/filebeat/inputs.d/webapp-non-ssl-access.yml':
ensure => absent,
}
profile::filebeat::log_input { "${vhost_name}-non-ssl-access":
paths => [ "/var/log/apache2/${vhost_name}_non-ssl_access.log" ],
fields => {
'apache_log_type' => 'access_log',
'environment' => $environment,
'vhost' => $vhost_name,
'application' => 'webapp',
},
}
- $filename_refresh_status = "refresh-savecodenow-statuses"
+ $filename_refresh_status = 'refresh-savecodenow-statuses'
$filepath_refresh_status = "/usr/local/bin/${filename_refresh_status}"
file {$filepath_refresh_status:
- ensure => present,
- owner => 'root',
- group => 'www-data',
- mode => '0755',
+ ensure => present,
+ owner => 'root',
+ group => 'www-data',
+ mode => '0755',
content => template("profile/swh/deploy/webapp/${filename_refresh_status}.sh.erb"),
}
$activate_once_per_environment_webapp = lookup('swh::deploy::webapp::cron::refresh_statuses')
if $activate_once_per_environment_webapp {
profile::cron::d {$filename_refresh_status:
target => $filename_refresh_status,
command => "chronic sh -c '${filepath_refresh_status}'",
minute => '*',
hour => '*',
}
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/base.pp b/site-modules/profile/manifests/swh/deploy/worker/base.pp
index dedfa3cc..c5361c5a 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/base.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/base.pp
@@ -1,45 +1,45 @@
# Base worker profile
class profile::swh::deploy::worker::base {
$systemd_template_unit_name = 'swh-worker@.service'
$systemd_unit_name = 'swh-worker.service'
$systemd_slice_name = 'system-swh\x2dworker.slice'
$celery_hostname = $swh_hostname['internal_fqdn']
package {'python3-swh.scheduler':
ensure => installed,
}
::systemd::unit_file {$systemd_template_unit_name:
- ensure => 'present',
+ ensure => 'present',
content => template("profile/swh/deploy/worker/${systemd_template_unit_name}.erb"),
}
::systemd::unit_file {$systemd_unit_name:
ensure => 'present',
source => "puppet:///modules/profile/swh/deploy/worker/${systemd_unit_name}",
} ~> service {'swh-worker':
ensure => running,
enable => true,
}
::systemd::unit_file {$systemd_slice_name:
ensure => 'present',
source => "puppet:///modules/profile/swh/deploy/worker/${systemd_slice_name}",
}
profile::cron::d {'cleanup-workers-tmp':
command => 'find /tmp -depth -mindepth 3 -maxdepth 3 -type d -ctime +2 -exec rm -rf {} \+',
target => 'swh-worker',
minute => 'fqdn_rand',
hour => 'fqdn_rand/2',
}
file {'/usr/local/sbin/swh-worker-ping-restart':
source => 'puppet:///modules/profile/swh/deploy/worker/swh-worker-ping-restart',
owner => 'root',
group => 'root',
mode => '0755',
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/checker_deposit.pp b/site-modules/profile/manifests/swh/deploy/worker/checker_deposit.pp
index 46b0927d..ce7cc95b 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/checker_deposit.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/checker_deposit.pp
@@ -1,15 +1,15 @@
# Deployment for deposit's archive checker
class profile::swh::deploy::worker::checker_deposit {
$packages = ['python3-swh.deposit.loader']
package {$packages:
ensure => 'present',
}
$private_tmp = lookup('swh::deploy::worker::checker_deposit::private_tmp')
::profile::swh::deploy::worker::instance {'checker_deposit':
- ensure => 'present',
- sentry_name => 'deposit',
- private_tmp => $private_tmp,
+ ensure => 'present',
+ sentry_name => 'deposit',
+ private_tmp => $private_tmp,
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
index e69727fb..c1b77e9e 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
@@ -1,13 +1,13 @@
# Deployment for swh-indexer-mimetype
class profile::swh::deploy::worker::indexer_content_mimetype {
include ::profile::swh::deploy::indexer
Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_mimetype':
- ensure => present,
- sentry_name => 'indexer',
- require => [
+ ensure => present,
+ sentry_name => 'indexer',
+ require => [
Class['profile::swh::deploy::indexer']
],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
index 580492f8..3e4caf2e 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
@@ -1,20 +1,20 @@
# Deployment for swh-indexer-fossology-license
class profile::swh::deploy::worker::indexer_fossology_license {
include ::profile::swh::deploy::indexer
$packages = ['fossology-nomossa']
package {$packages:
ensure => 'present',
require => Apt::Source['softwareheritage'],
}
Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_fossology_license':
- ensure => present,
- sentry_name => 'indexer',
- require => [
+ ensure => present,
+ sentry_name => 'indexer',
+ require => [
Class['profile::swh::deploy::indexer'],
Package[$packages],
],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
index ac201309..93e3004d 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
@@ -1,13 +1,13 @@
# Deployment for swh-indexer-origin-intrinsic-metadata
class profile::swh::deploy::worker::indexer_origin_intrinsic_metadata {
include ::profile::swh::deploy::indexer
Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_origin_intrinsic_metadata':
- ensure => present,
- sentry_name => 'indexer',
- require => [
+ ensure => present,
+ sentry_name => 'indexer',
+ require => [
Class['profile::swh::deploy::indexer'],
],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp
index 65ef2866..2078d85e 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp
@@ -1,13 +1,13 @@
# Deployment for swh-indexer-rehash
class profile::swh::deploy::worker::indexer_rehash {
include ::profile::swh::deploy::indexer
Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_rehash':
- ensure => 'stopped',
- sentry_name => 'indexer',
- require => [
+ ensure => 'stopped',
+ sentry_name => 'indexer',
+ require => [
Class['profile::swh::deploy::indexer']
],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_archive.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_archive.pp
index 8d8cdbf5..36b4a07e 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_archive.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_archive.pp
@@ -1,16 +1,16 @@
# Deployment for loader-archive
class profile::swh::deploy::worker::loader_archive {
include ::profile::swh::deploy::worker::base_loader_archive
$private_tmp = lookup('swh::deploy::worker::loader_archive::private_tmp')
::profile::swh::deploy::worker::instance {'loader_archive':
- ensure => present,
- private_tmp => $private_tmp,
- sentry_name => 'loader_core',
- require => [
+ ensure => present,
+ private_tmp => $private_tmp,
+ sentry_name => 'loader_core',
+ require => [
Package[$packages],
Package['lzip'],
],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_cran.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_cran.pp
index 7ab158be..3419e902 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_cran.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_cran.pp
@@ -1,16 +1,16 @@
# Deployment for loader-cran
class profile::swh::deploy::worker::loader_cran {
include ::profile::swh::deploy::worker::base_loader_archive
$private_tmp = lookup('swh::deploy::worker::loader_cran::private_tmp')
::profile::swh::deploy::worker::instance {'loader_cran':
- ensure => present,
- private_tmp => $private_tmp,
- sentry_name => 'loader_core',
- require => [
+ ensure => present,
+ private_tmp => $private_tmp,
+ sentry_name => 'loader_core',
+ require => [
Package[$packages],
Package['lzip'],
],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp
index b199fa47..11c50e8e 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp
@@ -1,20 +1,20 @@
# Deployment for loader-debian
class profile::swh::deploy::worker::loader_debian {
include ::profile::swh::deploy::worker::loader_package
package {'dpkg-dev':
ensure => 'present',
}
$private_tmp = lookup('swh::deploy::worker::loader_debian::private_tmp')
::profile::swh::deploy::worker::instance {'loader_debian':
- ensure => present,
- sentry_name => 'loader_core',
- private_tmp => $private_tmp,
- require => [
+ ensure => present,
+ sentry_name => 'loader_core',
+ private_tmp => $private_tmp,
+ require => [
Package[$packages],
Package['dpkg-dev'],
],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp
index 8dd1e74e..c528d060 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp
@@ -1,11 +1,11 @@
# Deployment for deposit's loader
class profile::swh::deploy::worker::loader_deposit {
include ::profile::swh::deploy::worker::loader_package
$private_tmp = lookup('swh::deploy::worker::loader_deposit::private_tmp')
::profile::swh::deploy::worker::instance {'loader_deposit':
- ensure => 'present',
- sentry_name => 'loader_core',
- private_tmp => $private_tmp,
+ ensure => 'present',
+ sentry_name => 'loader_core',
+ private_tmp => $private_tmp,
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp
index 11d18101..a09155e6 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp
@@ -1,11 +1,11 @@
# Deployment for swh-loader-git (remote)
class profile::swh::deploy::worker::loader_git {
include ::profile::swh::deploy::base_loader_git
::profile::swh::deploy::worker::instance {'loader_git':
- ensure => present,
- require => [
+ ensure => present,
+ require => [
Package[$::profile::swh::deploy::base_loader_git::packages],
],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp
index 1e9cfa7d..cca9f1d8 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp
@@ -1,14 +1,14 @@
# Deployment for high priority loader
class profile::swh::deploy::worker::loader_high_priority {
include ::profile::swh::deploy::base_loader_git
include ::profile::swh::deploy::base_loader_mercurial
include ::profile::swh::deploy::base_loader_svn
$packages = $::profile::swh::deploy::base_loader_git::packages + $::profile::swh::deploy::base_loader_mercurial::packages + $::profile::swh::deploy::base_loader_svn::packages
::profile::swh::deploy::worker::instance {'loader_high_priority':
- ensure => present,
- require => Package[$packages],
+ ensure => present,
+ require => Package[$packages],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp
index e956d742..101017a3 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp
@@ -1,13 +1,13 @@
# Deployment for swh-loader-mercurial (disk)
class profile::swh::deploy::worker::loader_mercurial {
include ::profile::swh::deploy::base_loader_mercurial
$private_tmp = lookup('swh::deploy::worker::loader_mercurial::private_tmp')
::profile::swh::deploy::worker::instance {'loader_mercurial':
- ensure => 'present',
- private_tmp => $private_tmp,
- require => [
+ ensure => 'present',
+ private_tmp => $private_tmp,
+ require => [
Package[$::profile::swh::deploy::base_loader_mercurial::packages],
],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_nixguix.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_nixguix.pp
index a84f157a..ffe837cb 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_nixguix.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_nixguix.pp
@@ -1,12 +1,12 @@
# Deployment for loader-nixguix
class profile::swh::deploy::worker::loader_nixguix {
include ::profile::swh::deploy::worker::base_loader_archive
$private_tmp = lookup('swh::deploy::worker::loader_nixguix::private_tmp')
::profile::swh::deploy::worker::instance {'loader_nixguix':
- ensure => present,
- private_tmp => $private_tmp,
- sentry_name => 'loader_core',
+ ensure => present,
+ private_tmp => $private_tmp,
+ sentry_name => 'loader_core',
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_npm.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_npm.pp
index a55a9967..fbec23a2 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_npm.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_npm.pp
@@ -1,19 +1,19 @@
# Deployment for swh-loader-npm
class profile::swh::deploy::worker::loader_npm {
$private_tmp = lookup('swh::deploy::worker::loader_npm::private_tmp')
$packages = ['python3-swh.loader.npm']
package {$packages:
ensure => 'present',
}
::profile::swh::deploy::worker::instance {'loader_npm':
- ensure => present,
- private_tmp => $private_tmp,
- sentry_name => 'loader_core',
- require => [
+ ensure => present,
+ private_tmp => $private_tmp,
+ sentry_name => 'loader_core',
+ require => [
Package[$packages],
],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp
index 445e7cc3..892bf85c 100644
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp
@@ -1,19 +1,19 @@
# Deployment for swh-loader-pypi
class profile::swh::deploy::worker::loader_pypi {
$private_tmp = lookup('swh::deploy::worker::loader_pypi::private_tmp')
$packages = ['python3-swh.loader.pypi']
package {$packages:
ensure => 'present',
}
::profile::swh::deploy::worker::instance {'loader_pypi':
- ensure => present,
- private_tmp => $private_tmp,
- sentry_name => 'loader_core',
- require => [
+ ensure => present,
+ private_tmp => $private_tmp,
+ sentry_name => 'loader_core',
+ require => [
Package[$packages],
],
}
}
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Sat, Jun 21, 7:00 PM (2 w, 3 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3268350
Attached To
rSPSITE puppet-swh-site
Event Timeline
Log In to Comment