diff --git a/manifests/swh/deploy/archiver_content_updater.pp b/manifests/swh/deploy/archiver_content_updater.pp index 3ded5be0..8cb1ccbe 100644 --- a/manifests/swh/deploy/archiver_content_updater.pp +++ b/manifests/swh/deploy/archiver_content_updater.pp @@ -1,47 +1,47 @@ # Deployment of the swh.storage.archiver.updater class profile::swh::deploy::archiver_content_updater { include profile::swh::deploy::archiver $conf_file = hiera('swh::deploy::archiver_content_updater::conf_file') $user = hiera('swh::deploy::archiver_content_updater::user') $group = hiera('swh::deploy::archiver_content_updater::group') $content_updater_config = hiera('swh::deploy::archiver_content_updater::config') include ::systemd $service_name = 'swh-archiver-content-updater' $service_file = "/etc/systemd/system/${service_name}.service" file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', - content => inline_template('<%= @content_updater_config.to_yaml %>'), + content => inline_template('<%= @content_updater_config.to_yaml %>\n'), notify => Service[$service_name], } # Template uses variables # - $user # - $group # file {$service_file: ensure => present, owner => 'root', group => 'root', mode => '0644', content => template('profile/swh/deploy/archiver/swh-content-updater.service.erb'), notify => [ Exec['systemd-daemon-reload'], Service[$service_name], ], } service {$service_name: ensure => running, enable => false, require => File[$service_file], } } diff --git a/manifests/swh/deploy/indexer.pp b/manifests/swh/deploy/indexer.pp index 0e3facc8..582c3c0a 100644 --- a/manifests/swh/deploy/indexer.pp +++ b/manifests/swh/deploy/indexer.pp @@ -1,34 +1,34 @@ # Base class for the indexer manifests class profile::swh::deploy::indexer { include ::profile::swh::deploy::objstorage_cloud $config_directory = '/etc/softwareheritage/indexer' $config_file = "${config_directory}/base.yml" $config = hiera('swh::deploy::worker::swh_indexer::base::config') $objstorage_config = hiera('swh::azure_objstorage::config') $merged_config = merge($config, {'objstorage' => $objstorage_config}) $packages = ['python3-swh.indexer'] file {$config_directory: ensure => 'directory', owner => 'swhworker', group => 'swhworker', mode => '0755', } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template('<%= @merged_config.to_yaml %>'), + content => inline_template('<%= @merged_config.to_yaml %>\n'), } package {$packages: ensure => 'latest', } } diff --git a/manifests/swh/deploy/journal_publisher.pp b/manifests/swh/deploy/journal_publisher.pp index ded78608..42ceeb95 100644 --- a/manifests/swh/deploy/journal_publisher.pp +++ b/manifests/swh/deploy/journal_publisher.pp @@ -1,57 +1,57 @@ # Deployment of the swh.journal.publisher class profile::swh::deploy::journal_publisher { include ::profile::swh::deploy::journal $conf_file = hiera('swh::deploy::journal_publisher::conf_file') $user = hiera('swh::deploy::journal_publisher::user') $group = hiera('swh::deploy::journal_publisher::group') $publisher_config = hiera('swh::deploy::journal_publisher::config') include ::systemd $service_name = 'swh-journal-publisher' $service_file = "/etc/systemd/system/${service_name}.service" file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0750', } file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', require => File[$conf_directory], - content => inline_template('<%= @publisher_config.to_yaml %>'), + content => inline_template('<%= @publisher_config.to_yaml %>\n'), notify => Service[$service_name], } # Template uses variables # - $user # - $group # file {$service_file: ensure => present, owner => 'root', group => 'root', mode => '0644', content => template('profile/swh/deploy/journal/swh-journal-publisher.service.erb'), require => Package[$package_name], notify => [ Exec['systemd-daemon-reload'], Service[$service_name], ], } service {$service_name: ensure => running, enable => false, require => File[$service_file], } } diff --git a/manifests/swh/deploy/journal_simple_checker_producer.pp b/manifests/swh/deploy/journal_simple_checker_producer.pp index 67e307e3..07161dc9 100644 --- a/manifests/swh/deploy/journal_simple_checker_producer.pp +++ b/manifests/swh/deploy/journal_simple_checker_producer.pp @@ -1,57 +1,57 @@ # Deployment of the swh.journal.checker class profile::swh::deploy::journal_simple_checker_producer { include ::profile::swh::deploy::journal $conf_file = hiera('swh::deploy::journal_simple_checker_producer::conf_file') $user = hiera('swh::deploy::journal_simple_checker_producer::user') $group = hiera('swh::deploy::journal_simple_checker_producer::group') $checker_config = hiera( 'swh::deploy::journal_simple_checker_producer::config') include ::systemd $service_name = 'swh-journal-simple-checker-producer' $service_file = "/etc/systemd/system/${service_name}.service" file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0750', } file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', require => File[$conf_directory], - content => inline_template('<%= @checker_config.to_yaml %>'), + content => inline_template('<%= @checker_config.to_yaml %>\n'), notify => Service[$service_name], } # Template uses variables # - $user # - $group # file {$service_file: ensure => present, owner => 'root', group => 'root', mode => '0644', content => template('profile/swh/deploy/journal/swh-journal-simple-checker-producer.service.erb'), require => Package[$package_name], notify => [ Exec['systemd-daemon-reload'], Service[$service_name], ], } service {$service_name: ensure => running, enable => false, require => File[$service_file], } } diff --git a/manifests/swh/deploy/objstorage.pp b/manifests/swh/deploy/objstorage.pp index fea323a8..2eafd794 100644 --- a/manifests/swh/deploy/objstorage.pp +++ b/manifests/swh/deploy/objstorage.pp @@ -1,74 +1,74 @@ # Deployment of the swh.objstorage.api server class profile::swh::deploy::objstorage { $conf_directory = hiera('swh::deploy::objstorage::conf_directory') $conf_file = hiera('swh::deploy::objstorage::conf_file') $user = hiera('swh::deploy::objstorage::user') $group = hiera('swh::deploy::objstorage::group') $objstorage_config = hiera('swh::deploy::objstorage::config') $swh_packages = ['python3-swh.objstorage'] $uwsgi_listen_address = hiera('swh::deploy::objstorage::uwsgi::listen') $uwsgi_workers = hiera('swh::deploy::objstorage::uwsgi::workers') $uwsgi_http_workers = hiera('swh::deploy::objstorage::uwsgi::http_workers') $uwsgi_http_keepalive = hiera('swh::deploy::objstorage::uwsgi::http_keepalive') $uwsgi_http_timeout = hiera('swh::deploy::objstorage::uwsgi::http_timeout') $uwsgi_max_requests = hiera('swh::deploy::objstorage::uwsgi::max_requests') $uwsgi_max_requests_delta = hiera('swh::deploy::objstorage::uwsgi::max_requests_delta') $uwsgi_reload_mercy = hiera('swh::deploy::objstorage::uwsgi::reload_mercy') include ::uwsgi package {$swh_packages: ensure => latest, require => Apt::Source['softwareheritage'], } file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0750', } file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', - content => inline_template('<%= @objstorage_config.to_yaml %>'), + content => inline_template('<%= @objstorage_config.to_yaml %>\n'), } ::uwsgi::site {'swh-objstorage': ensure => enabled, settings => { plugin => 'python3', workers => $uwsgi_workers, max_requests => $uwsgi_max_requests, max_requests_delta => $uwsgi_max_requests_delta, worker_reload_mercy => $uwsgi_reload_mercy, reload_mercy => $uwsgi_reload_mercy, uid => $user, gid => $user, umask => '022', module => 'swh.objstorage.api.server', callable => 'run_from_webserver', } } ::uwsgi::site {'swh-objstorage-http': ensure => enabled, settings => { workers => 0, http => $uwsgi_listen_address, http_workers => $uwsgi_http_workers, http_keepalive => $uwsgi_http_keepalive, http_timeout => $uwsgi_http_timeout, http_to => '/var/run/uwsgi/app/swh-objstorage/socket', uid => $user, gid => $user, } } } diff --git a/manifests/swh/deploy/objstorage_archive_notifier_checker.pp b/manifests/swh/deploy/objstorage_archive_notifier_checker.pp index 324052e1..acccc8da 100644 --- a/manifests/swh/deploy/objstorage_archive_notifier_checker.pp +++ b/manifests/swh/deploy/objstorage_archive_notifier_checker.pp @@ -1,49 +1,49 @@ # Deployment of the swh.objstorage.checker.ArchiveNotifierContentChecker class profile::swh::deploy::objstorage_archive_notifier_checker { $conf_directory = hiera('swh::deploy::objstorage_archive_notifier_checker::conf_directory') $conf_file = hiera('swh::deploy::objstorage_archive_notifier_checker::conf_file') $user = hiera('swh::deploy::objstorage_archive_notifier_checker::user') $group = hiera('swh::deploy::objstorage_archive_notifier_checker::group') # configuration file $archive_notifier_config = hiera('swh::deploy::objstorage_archive_notifier_checker::config') $swh_packages = ['python3-swh.objstorage.checker'] package {$swh_packages: ensure => latest, require => Apt::Source['softwareheritage'], } file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0750', } file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', - content => inline_template('<%= @archive_notifier_config.to_yaml %>'), + content => inline_template('<%= @archive_notifier_config.to_yaml %>\n'), } include ::systemd file {'/etc/systemd/system/objstorage_archive_notifier_checker.service': ensure => present, owner => 'root', group => 'root', mode => '0644', content => template('profile/swh/deploy/storage/objstorage_archive_notifier_checker.service.erb'), notify => Exec['systemd-daemon-reload'], require => [ File[$conf_file], Package[$swh_packages], ] } } diff --git a/manifests/swh/deploy/objstorage_log_checker.pp b/manifests/swh/deploy/objstorage_log_checker.pp index ad3b1fee..3012e806 100644 --- a/manifests/swh/deploy/objstorage_log_checker.pp +++ b/manifests/swh/deploy/objstorage_log_checker.pp @@ -1,49 +1,49 @@ # Deployment of the swh.objstorage.checker.LogContentChecker class profile::swh::deploy::objstorage_log_checker { $conf_directory = hiera('swh::deploy::objstorage_log_checker::conf_directory') $conf_file = hiera('swh::deploy::objstorage_log_checker::conf_file') $user = hiera('swh::deploy::objstorage_log_checker::user') $group = hiera('swh::deploy::objstorage_log_checker::group') # configuration file $log_checker_config = hiera('swh::deploy::objstorage_log_checker::config') $swh_packages = ['python3-swh.objstorage.checker'] package {$swh_packages: ensure => latest, require => Apt::Source['softwareheritage'], } file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0750', } file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', - content => inline_template('<%= @log_checker_config.to_yaml %>'), + content => inline_template('<%= @log_checker_config.to_yaml %>\n'), } include ::systemd file {'/etc/systemd/system/objstorage_log_checker.service': ensure => present, owner => 'root', group => 'root', mode => '0644', content => template('profile/swh/deploy/storage/objstorage_log_checker.service.erb'), notify => Exec['systemd-daemon-reload'], require => [ File[$conf_file], Package[$swh_packages], ] } } diff --git a/manifests/swh/deploy/objstorage_repair_checker.pp b/manifests/swh/deploy/objstorage_repair_checker.pp index a907fcc7..378ce8ff 100644 --- a/manifests/swh/deploy/objstorage_repair_checker.pp +++ b/manifests/swh/deploy/objstorage_repair_checker.pp @@ -1,48 +1,48 @@ # Deployment of the swh.objstorage.checker.RepairContentChecker class profile::swh::deploy::objstorage_repair_checker { $conf_directory = hiera('swh::deploy::objstorage_repair_checker::conf_directory') $conf_file = hiera('swh::deploy::objstorage_repair_checker::conf_file') $user = hiera('swh::deploy::objstorage_repair_checker::user') $group = hiera('swh::deploy::objstorage_repair_checker::group') $repair_checker_config = hiera('swh::deploy::objstorage_repair_checker::config') $swh_packages = ['python3-swh.objstorage.checker'] package {$swh_packages: ensure => latest, require => Apt::Source['softwareheritage'], } file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0750', } file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', - content => inline_template('<%= @repair_checker_config.to_yaml %>'), + content => inline_template('<%= @repair_checker_config.to_yaml %>\n'), } include ::systemd file {'/etc/systemd/system/objstorage_repair_checker.service': ensure => present, owner => 'root', group => 'root', mode => '0644', content => template('profile/swh/deploy/storage/objstorage_repair_checker.service.erb'), notify => Exec['systemd-daemon-reload'], require => [ File[$conf_file], Package[$swh_packages], ] } } diff --git a/manifests/swh/deploy/storage.pp b/manifests/swh/deploy/storage.pp index d64c7e82..6b6342c9 100644 --- a/manifests/swh/deploy/storage.pp +++ b/manifests/swh/deploy/storage.pp @@ -1,76 +1,76 @@ # Deployment of the swh.storage.api server class profile::swh::deploy::storage { $conf_directory = hiera('swh::deploy::storage::conf_directory') $conf_file = hiera('swh::deploy::storage::conf_file') $user = hiera('swh::deploy::storage::user') $group = hiera('swh::deploy::storage::group') $swh_packages = ['python3-swh.storage'] $uwsgi_listen_address = hiera('swh::deploy::storage::uwsgi::listen') $uwsgi_workers = hiera('swh::deploy::storage::uwsgi::workers') $uwsgi_http_workers = hiera('swh::deploy::storage::uwsgi::http_workers') $uwsgi_http_keepalive = hiera('swh::deploy::storage::uwsgi::http_keepalive') $uwsgi_http_timeout = hiera('swh::deploy::storage::uwsgi::http_timeout') $uwsgi_max_requests = hiera('swh::deploy::storage::uwsgi::max_requests') $uwsgi_max_requests_delta = hiera('swh::deploy::storage::uwsgi::max_requests_delta') $uwsgi_reload_mercy = hiera('swh::deploy::storage::uwsgi::reload_mercy') $storage_config = hiera('swh::deploy::storage::config') include ::uwsgi package {$swh_packages: ensure => latest, require => Apt::Source['softwareheritage'], notify => Service['uwsgi'], } file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0755', } file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', - content => inline_template('<%= @storage_config.to_yaml %>'), + content => inline_template('<%= @storage_config.to_yaml %>\n'), notify => Service['uwsgi'], } ::uwsgi::site {'swh-storage': ensure => enabled, settings => { plugin => 'python3', workers => $uwsgi_workers, max_requests => $uwsgi_max_requests, max_requests_delta => $uwsgi_max_requests_delta, worker_reload_mercy => $uwsgi_reload_mercy, reload_mercy => $uwsgi_reload_mercy, uid => $user, gid => $user, umask => '022', module => 'swh.storage.api.server', callable => 'run_from_webserver', } } ::uwsgi::site {'swh-storage-http': ensure => enabled, settings => { workers => 0, http => $uwsgi_listen_address, http_workers => $uwsgi_http_workers, http_keepalive => $uwsgi_http_keepalive, http_timeout => $uwsgi_http_timeout, http_to => '/var/run/uwsgi/app/swh-storage/socket', uid => $user, gid => $user, } } } diff --git a/manifests/swh/deploy/webapp.pp b/manifests/swh/deploy/webapp.pp index 7494ce01..5b672417 100644 --- a/manifests/swh/deploy/webapp.pp +++ b/manifests/swh/deploy/webapp.pp @@ -1,180 +1,180 @@ # WebApp deployment class profile::swh::deploy::webapp { $conf_directory = hiera('swh::deploy::webapp::conf_directory') $conf_file = hiera('swh::deploy::webapp::conf_file') $user = hiera('swh::deploy::webapp::user') $group = hiera('swh::deploy::webapp::group') $webapp_config = hiera('swh::deploy::webapp::config') $conf_log_dir = hiera('swh::deploy::webapp::conf::log_dir') $uwsgi_listen_address = hiera('swh::deploy::webapp::uwsgi::listen') $uwsgi_protocol = hiera('swh::deploy::webapp::uwsgi::protocol') $uwsgi_workers = hiera('swh::deploy::webapp::uwsgi::workers') $uwsgi_max_requests = hiera('swh::deploy::webapp::uwsgi::max_requests') $uwsgi_max_requests_delta = hiera('swh::deploy::webapp::uwsgi::max_requests_delta') $uwsgi_reload_mercy = hiera('swh::deploy::webapp::uwsgi::reload_mercy') $swh_packages = ['python3-swh.web.ui'] $static_dir = '/usr/lib/python3/dist-packages/swh/web/ui/static' $vhost_name = hiera('swh::deploy::webapp::vhost::name') $vhost_aliases = hiera('swh::deploy::webapp::vhost::aliases') $vhost_docroot = hiera('swh::deploy::webapp::vhost::docroot') $vhost_basic_auth_file = "${conf_directory}/http_auth" $vhost_basic_auth_content = hiera('swh::deploy::webapp::vhost::basic_auth_content') $vhost_ssl_protocol = hiera('swh::deploy::webapp::vhost::ssl_protocol') $vhost_ssl_honorcipherorder = hiera('swh::deploy::webapp::vhost::ssl_honorcipherorder') $vhost_ssl_cipher = hiera('swh::deploy::webapp::vhost::ssl_cipher') $locked_endpoints = hiera_array('swh::deploy::webapp::locked_endpoints') $endpoint_directories = $locked_endpoints.map |$endpoint| { { path => "^${endpoint}", provider => 'locationmatch', auth_type => 'Basic', auth_name => 'Software Heritage development', auth_user_file => $vhost_basic_auth_file, auth_require => 'valid-user', } } include ::uwsgi package {$swh_packages: ensure => latest, require => Apt::Source['softwareheritage'], notify => Service['uwsgi'], } file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0755', } file {$conf_log_dir: ensure => directory, owner => 'root', group => $group, mode => '0770', } file {$vhost_docroot: ensure => directory, owner => 'root', group => $group, mode => '0755', } file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', - content => inline_template('<%= @webapp_config.to_yaml %>'), + content => inline_template('<%= @webapp_config.to_yaml %>\n'), notify => Service['uwsgi'], } ::uwsgi::site {'swh-webapp': ensure => enabled, settings => { plugin => 'python3', protocol => $uwsgi_protocol, socket => $uwsgi_listen_address, workers => $uwsgi_workers, max_requests => $uwsgi_max_requests, max_requests_delta => $uwsgi_max_requests_delta, worker_reload_mercy => $uwsgi_reload_mercy, reload_mercy => $uwsgi_reload_mercy, uid => $user, gid => $user, umask => '022', module => 'swh.web.ui.main', callable => 'run_from_webserver', } } include ::profile::ssl include ::profile::apache::common include ::apache::mod::proxy include ::apache::mod::headers ::apache::mod {'proxy_uwsgi':} ::apache::vhost {"${vhost_name}_non-ssl": servername => $vhost_name, serveraliases => $vhost_aliases, port => '80', docroot => $vhost_docroot, redirect_status => 'permanent', redirect_dest => "https://${vhost_name}/", } $ssl_cert_name = 'star_softwareheritage_org' $ssl_cert = $::profile::ssl::certificate_paths[$ssl_cert_name] $ssl_ca = $::profile::ssl::ca_paths[$ssl_cert_name] $ssl_key = $::profile::ssl::private_key_paths[$ssl_cert_name] ::apache::vhost {"${vhost_name}_ssl": servername => $vhost_name, serveraliases => $vhost_aliases, port => '443', ssl => true, ssl_protocol => $vhost_ssl_protocol, ssl_honorcipherorder => $vhost_ssl_honorcipherorder, ssl_cipher => $vhost_ssl_cipher, ssl_cert => $ssl_cert, ssl_ca => $ssl_ca, ssl_key => $ssl_key, docroot => $vhost_docroot, proxy_pass => [ { path => '/static', url => '!', }, { path => '/robots.txt', url => '!', }, { path => '/favicon.ico', url => '!', }, { path => '/', url => "uwsgi://${uwsgi_listen_address}/", }, ], directories => [ { path => '/api', provider => 'location', allow => 'from all', satisfy => 'Any', headers => ['add Access-Control-Allow-Origin "*"'], }, { path => $static_dir, options => ['-Indexes'], }, ] + $endpoint_directories, aliases => [ { alias => '/static', path => $static_dir, }, { alias => '/robots.txt', path => "${static_dir}/robots.txt", }, ], require => [ File[$vhost_basic_auth_file], File[$ssl_cert], File[$ssl_ca], File[$ssl_key], ], } file {$vhost_basic_auth_file: ensure => present, owner => 'root', group => 'www-data', mode => '0640', content => $vhost_basic_auth_content, } } diff --git a/manifests/swh/deploy/worker/swh_indexer_ctags.pp b/manifests/swh/deploy/worker/swh_indexer_ctags.pp index 6b4cc3b9..8c7a3bae 100644 --- a/manifests/swh/deploy/worker/swh_indexer_ctags.pp +++ b/manifests/swh/deploy/worker/swh_indexer_ctags.pp @@ -1,41 +1,41 @@ # Deployment for swh-indexer-ctags class profile::swh::deploy::worker::swh_indexer_ctags { include ::profile::swh::deploy::indexer $concurrency = hiera('swh::deploy::worker::swh_indexer::ctags::concurrency') $loglevel = hiera('swh::deploy::worker::swh_indexer::ctags::loglevel') $task_broker = hiera('swh::deploy::worker::swh_indexer::ctags::task_broker') $config_file = '/etc/softwareheritage/indexer/ctags.yml' $config = hiera('swh::deploy::worker::swh_indexer::ctags::config') $objstorage_config = hiera('swh::azure_objstorage::config') $merged_config = merge($config, {'objstorage' => $objstorage_config}) $task_modules = ['swh.indexer.tasks'] $task_queues = ['swh_indexer_content_ctags'] ::profile::swh::deploy::worker::instance {'swh_indexer_ctags': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template('<%= @merged_config.to_yaml %>'), + content => inline_template('<%= @merged_config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp b/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp index ffa29572..27b017ac 100644 --- a/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp +++ b/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp @@ -1,41 +1,41 @@ # Deployment for swh-indexer-fossology-license class profile::swh::deploy::worker::swh_indexer_fossology_license { include ::profile::swh::deploy::indexer $concurrency = hiera('swh::deploy::worker::swh_indexer::fossology_license::concurrency') $loglevel = hiera('swh::deploy::worker::swh_indexer::fossology_license::loglevel') $task_broker = hiera('swh::deploy::worker::swh_indexer::fossology_license::task_broker') $config_file = '/etc/softwareheritage/indexer/fossology_license.yml' $config = hiera('swh::deploy::worker::swh_indexer::fossology_license::config') $objstorage_config = hiera('swh::azure_objstorage::config') $merged_config = merge($config, {'objstorage' => $objstorage_config}) $task_modules = ['swh.indexer.tasks'] $task_queues = ['swh_indexer_content_fossology_license'] ::profile::swh::deploy::worker::instance {'swh_indexer_fossology_license': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template('<%= @merged_config.to_yaml %>'), + content => inline_template('<%= @merged_config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_indexer_language.pp b/manifests/swh/deploy/worker/swh_indexer_language.pp index 99f6077d..91dd2577 100644 --- a/manifests/swh/deploy/worker/swh_indexer_language.pp +++ b/manifests/swh/deploy/worker/swh_indexer_language.pp @@ -1,41 +1,41 @@ # Deployment for swh-indexer-language class profile::swh::deploy::worker::swh_indexer_language { include ::profile::swh::deploy::indexer $concurrency = hiera('swh::deploy::worker::swh_indexer::language::concurrency') $loglevel = hiera('swh::deploy::worker::swh_indexer::language::loglevel') $task_broker = hiera('swh::deploy::worker::swh_indexer::language::task_broker') $config_file = '/etc/softwareheritage/indexer/language.yml' $config = hiera('swh::deploy::worker::swh_indexer::language::config') $objstorage_config = hiera('swh::azure_objstorage::config') $merged_config = merge($config, {'objstorage' => $objstorage_config}) $task_modules = ['swh.indexer.tasks'] $task_queues = ['swh_indexer_content_language'] ::profile::swh::deploy::worker::instance {'swh_indexer_language': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template('<%= @merged_config.to_yaml %>'), + content => inline_template('<%= @merged_config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_indexer_mimetype.pp b/manifests/swh/deploy/worker/swh_indexer_mimetype.pp index 3844a7bb..0b5f236b 100644 --- a/manifests/swh/deploy/worker/swh_indexer_mimetype.pp +++ b/manifests/swh/deploy/worker/swh_indexer_mimetype.pp @@ -1,41 +1,41 @@ # Deployment for swh-indexer-mimetype class profile::swh::deploy::worker::swh_indexer_mimetype { include ::profile::swh::deploy::indexer $concurrency = hiera('swh::deploy::worker::swh_indexer::mimetype::concurrency') $loglevel = hiera('swh::deploy::worker::swh_indexer::mimetype::loglevel') $task_broker = hiera('swh::deploy::worker::swh_indexer::mimetype::task_broker') $config_file = '/etc/softwareheritage/indexer/mimetype.yml' $config = hiera('swh::deploy::worker::swh_indexer::mimetype::config') $objstorage_config = hiera('swh::azure_objstorage::config') $merged_config = merge($config, {'objstorage' => $objstorage_config}) $task_modules = ['swh.indexer.tasks'] $task_queues = ['swh_indexer_content_mimetype'] ::profile::swh::deploy::worker::instance {'swh_indexer_mimetype': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template('<%= @merged_config.to_yaml %>'), + content => inline_template('<%= @merged_config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_indexer_orchestrator.pp b/manifests/swh/deploy/worker/swh_indexer_orchestrator.pp index cb2576ed..3155d228 100644 --- a/manifests/swh/deploy/worker/swh_indexer_orchestrator.pp +++ b/manifests/swh/deploy/worker/swh_indexer_orchestrator.pp @@ -1,37 +1,37 @@ # Deployment for swh-indexer class profile::swh::deploy::worker::swh_indexer_orchestrator { include ::profile::swh::deploy::indexer $concurrency = hiera('swh::deploy::worker::swh_indexer::orchestrator::concurrency') $loglevel = hiera('swh::deploy::worker::swh_indexer::orchestrator::loglevel') $task_broker = hiera('swh::deploy::worker::swh_indexer::orchestrator::task_broker') $config_file = '/etc/softwareheritage/indexer/orchestrator.yml' $config = hiera('swh::deploy::worker::swh_indexer::orchestrator::config') $task_modules = ['swh.indexer.tasks'] $task_queues = ['swh_indexer_orchestrator_content_all'] ::profile::swh::deploy::worker::instance {'swh_indexer_orchestrator': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhworker', mode => '0644', - content => inline_template('<%= @config.to_yaml %>'), + content => inline_template('<%= @config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_indexer_orchestrator_text.pp b/manifests/swh/deploy/worker/swh_indexer_orchestrator_text.pp index b6557355..a16ed882 100644 --- a/manifests/swh/deploy/worker/swh_indexer_orchestrator_text.pp +++ b/manifests/swh/deploy/worker/swh_indexer_orchestrator_text.pp @@ -1,37 +1,37 @@ # Deployment for swh-indexer class profile::swh::deploy::worker::swh_indexer_orchestrator_text { include ::profile::swh::deploy::indexer $concurrency = hiera('swh::deploy::worker::swh_indexer::orchestrator_text::concurrency') $loglevel = hiera('swh::deploy::worker::swh_indexer::orchestrator_text::loglevel') $task_broker = hiera('swh::deploy::worker::swh_indexer::orchestrator_text::task_broker') $config_file = '/etc/softwareheritage/indexer/orchestrator_text.yml' $config = hiera('swh::deploy::worker::swh_indexer::orchestrator_text::config') $task_modules = ['swh.indexer.tasks'] $task_queues = ['swh_indexer_orchestrator_content_text'] ::profile::swh::deploy::worker::instance {'swh_indexer_orchestrator_text': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhworker', mode => '0644', - content => inline_template('<%= @config.to_yaml %>'), + content => inline_template('<%= @config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_lister_github.pp b/manifests/swh/deploy/worker/swh_lister_github.pp index 5868de01..ab569384 100644 --- a/manifests/swh/deploy/worker/swh_lister_github.pp +++ b/manifests/swh/deploy/worker/swh_lister_github.pp @@ -1,40 +1,40 @@ # Deployment for swh-lister-github class profile::swh::deploy::worker::swh_lister_github { $concurrency = hiera('swh::deploy::worker::swh_lister_github::concurrency') $loglevel = hiera('swh::deploy::worker::swh_lister_github::loglevel') $task_broker = hiera('swh::deploy::worker::swh_lister_github::task_broker') $config_file = '/etc/softwareheritage/lister-github.yml' $config = hiera_hash('swh::deploy::worker::swh_lister_github::config') $task_modules = ['swh.lister.github.tasks'] $task_queues = ['swh_lister_github_incremental', 'swh_lister_github_full'] $packages = ['python3-swh.lister.github'] package {$packages: ensure => 'installed', } ::profile::swh::deploy::worker::instance {'swh_lister_github': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Package[$packages], File[$config_file], ], } # Contains passwords file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', mode => '0640', - content => inline_template('<%= @config.to_yaml %>'), + content => inline_template('<%= @config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_loader_git.pp b/manifests/swh/deploy/worker/swh_loader_git.pp index 5b9bd581..2e89ac94 100644 --- a/manifests/swh/deploy/worker/swh_loader_git.pp +++ b/manifests/swh/deploy/worker/swh_loader_git.pp @@ -1,35 +1,35 @@ # Deployment for swh-loader-git (remote) class profile::swh::deploy::worker::swh_loader_git { include ::profile::swh::deploy::base_loader_git $concurrency = hiera('swh::deploy::worker::swh_loader_git::concurrency') $loglevel = hiera('swh::deploy::worker::swh_loader_git::loglevel') $task_broker = hiera('swh::deploy::worker::swh_loader_git::task_broker') $config_file = '/etc/softwareheritage/loader/git-updater.yml' $config = hiera('swh::deploy::worker::swh_loader_git::config') $task_modules = ['swh.loader.git.tasks'] $task_queues = ['swh_loader_git'] ::profile::swh::deploy::worker::instance {'swh_loader_git': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::base_loader_git'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhworker', mode => '0644', - content => inline_template('<%= @config.to_yaml %>'), + content => inline_template('<%= @config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_loader_git_disk.pp b/manifests/swh/deploy/worker/swh_loader_git_disk.pp index 77d22ebe..5c5a07fa 100644 --- a/manifests/swh/deploy/worker/swh_loader_git_disk.pp +++ b/manifests/swh/deploy/worker/swh_loader_git_disk.pp @@ -1,35 +1,35 @@ # Deployment for swh-loader-git (disk) class profile::swh::deploy::worker::swh_loader_git_disk { include ::profile::swh::deploy::base_loader_git $concurrency = hiera('swh::deploy::worker::swh_loader_git_disk::concurrency') $loglevel = hiera('swh::deploy::worker::swh_loader_git_disk::loglevel') $task_broker = hiera('swh::deploy::worker::swh_loader_git_disk::task_broker') $config_file = '/etc/softwareheritage/loader/git-loader.yml' $config = hiera('swh::deploy::worker::swh_loader_git_disk::config') $task_modules = ['swh.loader.git.tasks'] $task_queues = ['swh_loader_git_express', 'swh_loader_git_archive'] ::profile::swh::deploy::worker::instance {'swh_loader_git_disk': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::base_loader_git'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhworker', mode => '0644', - content => inline_template('<%= @config.to_yaml %>'), + content => inline_template('<%= @config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_loader_svn.pp b/manifests/swh/deploy/worker/swh_loader_svn.pp index 7277a3fe..8db66b89 100644 --- a/manifests/swh/deploy/worker/swh_loader_svn.pp +++ b/manifests/swh/deploy/worker/swh_loader_svn.pp @@ -1,39 +1,39 @@ # Deployment for swh-loader-svn class profile::swh::deploy::worker::swh_loader_svn { $concurrency = hiera('swh::deploy::worker::swh_loader_svn::concurrency') $loglevel = hiera('swh::deploy::worker::swh_loader_svn::loglevel') $task_broker = hiera('swh::deploy::worker::swh_loader_svn::task_broker') $config_file = '/etc/softwareheritage/loader/svn.yml' $config = hiera('swh::deploy::worker::swh_loader_svn::config') $task_modules = ['swh.loader.svn.tasks'] $task_queues = ['swh_loader_svn', 'swh_loader_svn_mount_and_load'] $packages = ['python3-swh.loader.svn'] package {$packages: ensure => 'latest', } ::profile::swh::deploy::worker::instance {'swh_loader_svn': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Package[$packages], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhworker', mode => '0644', - content => inline_template('<%= @config.to_yaml %>'), + content => inline_template('<%= @config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_provenance_cache.pp b/manifests/swh/deploy/worker/swh_provenance_cache.pp index 52023e1e..466a921d 100644 --- a/manifests/swh/deploy/worker/swh_provenance_cache.pp +++ b/manifests/swh/deploy/worker/swh_provenance_cache.pp @@ -1,45 +1,45 @@ # Deployment for swh-provenance-cache class profile::swh::deploy::worker::swh_provenance_cache { include ::profile::swh::deploy::loader $concurrency = hiera('swh::deploy::worker::swh_provenance_cache::concurrency') $loglevel = hiera('swh::deploy::worker::swh_provenance_cache::loglevel') $task_broker = hiera('swh::deploy::worker::swh_provenance_cache::task_broker') $config_file = '/etc/softwareheritage/storage/provenance_cache.yml' $config = hiera('swh::deploy::worker::swh_provenance_cache::config') $task_modules = ['swh.storage.provenance.tasks'] $task_queues = [ 'swh_populate_cache_content_revision', 'swh_populate_cache_revision_origin' ] $packages = ['python3-swh.storage.provenance'] package {$packages: ensure => 'installed', } ::profile::swh::deploy::worker::instance {'provenance-cache': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Package[$packages], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains password mode => '0640', - content => inline_template('<%= @config.to_yaml %>'), + content => inline_template('<%= @config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_reader_git.pp b/manifests/swh/deploy/worker/swh_reader_git.pp index 9fec091e..bfa0afc6 100644 --- a/manifests/swh/deploy/worker/swh_reader_git.pp +++ b/manifests/swh/deploy/worker/swh_reader_git.pp @@ -1,37 +1,37 @@ # Deployment for swh-reader-git class profile::swh::deploy::worker::swh_reader_git { include ::profile::swh::deploy::base_loader_git include ::profile::swh::deploy::worker::swh_storage_archiver_azure $concurrency = hiera('swh::deploy::worker::swh_reader_git::concurrency') $loglevel = hiera('swh::deploy::worker::swh_reader_git::loglevel') $task_broker = hiera('swh::deploy::worker::swh_reader_git::task_broker') $config_file = '/etc/softwareheritage/loader/git-remote-reader.yml' $config = hiera('swh::deploy::worker::swh_reader_git::config') $task_modules = ['swh.loader.git.tasks'] $task_queues = ['swh_reader_git'] ::profile::swh::deploy::worker::instance {'swh_reader_git': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::base_loader_git'], Class['profile::swh::deploy::worker::swh_storage_archiver_azure'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhworker', mode => '0644', - content => inline_template('<%= @config.to_yaml %>'), + content => inline_template('<%= @config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_storage_archiver.pp b/manifests/swh/deploy/worker/swh_storage_archiver.pp index a7713f2d..3615c9ae 100644 --- a/manifests/swh/deploy/worker/swh_storage_archiver.pp +++ b/manifests/swh/deploy/worker/swh_storage_archiver.pp @@ -1,38 +1,38 @@ # Deployment for swh-storage-archiver class profile::swh::deploy::worker::swh_storage_archiver { include ::profile::swh::deploy::archiver $concurrency = hiera('swh::deploy::worker::swh_storage_archiver::concurrency') $loglevel = hiera('swh::deploy::worker::swh_storage_archiver::loglevel') $task_broker = hiera('swh::deploy::worker::swh_storage_archiver::task_broker') $config_file = hiera('swh::deploy::worker::swh_storage_archiver::conf_file') $config = hiera('swh::deploy::worker::swh_storage_archiver::config') $storages_config = hiera_array('swh::deploy::worker::swh_storage_archiver::storages') $merged_config = merge($config, {'storages' => $storages_config}) $task_modules = ['swh.storage.archiver.tasks'] $task_queues = ['swh_storage_archive_worker'] ::profile::swh::deploy::worker::instance {'swh_storage_archiver': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template('<%= @merged_config.to_yaml %>'), + content => inline_template('<%= @merged_config.to_yaml %>\n'), } } diff --git a/manifests/swh/deploy/worker/swh_storage_archiver_azure.pp b/manifests/swh/deploy/worker/swh_storage_archiver_azure.pp index 8d1bff79..5bbdf543 100644 --- a/manifests/swh/deploy/worker/swh_storage_archiver_azure.pp +++ b/manifests/swh/deploy/worker/swh_storage_archiver_azure.pp @@ -1,49 +1,49 @@ # Deployment for swh-storage-archiver-azure class profile::swh::deploy::worker::swh_storage_archiver_azure { include ::profile::swh::deploy::archiver include ::profile::swh::deploy::indexer $concurrency = hiera('swh::deploy::worker::swh_storage_archiver_azure::concurrency') $loglevel = hiera('swh::deploy::worker::swh_storage_archiver_azure::loglevel') $task_broker = hiera('swh::deploy::worker::swh_storage_archiver_azure::task_broker') $config_file = hiera('swh::deploy::worker::swh_storage_archiver_azure::conf_file') $config = hiera('swh::deploy::worker::swh_storage_archiver_azure::config') $storages_config_list = hiera_array( 'swh::deploy::worker::swh_storage_archiver::storages') $objstorage_azure_config = hiera('swh::azure_objstorage::config') $objstorages_config = $storages_config_list + [ merge({"host" => "azure"}, $objstorage_azure_config) ] # Create the full configuration $merged_config = merge($config, { 'storages' => $objstorages_config }) $task_modules = ['swh.storage.archiver.tasks', 'swh.indexer.tasks'] $task_queues = ['swh_storage_archive_worker_to_backend'] ::profile::swh::deploy::worker::instance {'swh_storage_archiver_azure': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template('<%= @merged_config.to_yaml %>'), + content => inline_template('<%= @merged_config.to_yaml %>\n'), } }