diff --git a/manifests/swh/deploy/indexer.pp b/manifests/swh/deploy/indexer.pp index 69a23700..f4a0ffc4 100644 --- a/manifests/swh/deploy/indexer.pp +++ b/manifests/swh/deploy/indexer.pp @@ -1,34 +1,31 @@ # Base class for the indexer manifests class profile::swh::deploy::indexer { include ::profile::swh::deploy::objstorage_cloud $config_directory = '/etc/softwareheritage/indexer' $config_file = "${config_directory}/base.yml" $config = hiera('swh::deploy::worker::swh_indexer::base::config') - $objstorage_config = hiera('swh::azure_objstorage::config') - $merged_config = merge($config, {'objstorage' => $objstorage_config}) - $packages = ['python3-swh.indexer'] file {$config_directory: ensure => 'directory', owner => 'swhworker', group => 'swhworker', mode => '0755', } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template("<%= @merged_config.to_yaml %>\n"), + content => inline_template("<%= @config.to_yaml %>\n"), } package {$packages: ensure => 'latest', } } diff --git a/manifests/swh/deploy/worker/swh_indexer_ctags.pp b/manifests/swh/deploy/worker/swh_indexer_ctags.pp index 876885ca..e83e17aa 100644 --- a/manifests/swh/deploy/worker/swh_indexer_ctags.pp +++ b/manifests/swh/deploy/worker/swh_indexer_ctags.pp @@ -1,41 +1,38 @@ # Deployment for swh-indexer-ctags class profile::swh::deploy::worker::swh_indexer_ctags { include ::profile::swh::deploy::indexer $concurrency = hiera('swh::deploy::worker::swh_indexer::ctags::concurrency') $loglevel = hiera('swh::deploy::worker::swh_indexer::ctags::loglevel') $task_broker = hiera('swh::deploy::worker::swh_indexer::ctags::task_broker') $config_file = '/etc/softwareheritage/indexer/ctags.yml' $config = hiera('swh::deploy::worker::swh_indexer::ctags::config') - $objstorage_config = hiera('swh::azure_objstorage::config') - $merged_config = merge($config, {'objstorage' => $objstorage_config}) - $task_modules = ['swh.indexer.tasks'] $task_queues = ['swh_indexer_content_ctags'] ::profile::swh::deploy::worker::instance {'swh_indexer_ctags': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template("<%= @merged_config.to_yaml %>\n"), + content => inline_template("<%= @config.to_yaml %>\n"), } } diff --git a/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp b/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp index b201ddd3..e6fb999a 100644 --- a/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp +++ b/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp @@ -1,41 +1,38 @@ # Deployment for swh-indexer-fossology-license class profile::swh::deploy::worker::swh_indexer_fossology_license { include ::profile::swh::deploy::indexer $concurrency = hiera('swh::deploy::worker::swh_indexer::fossology_license::concurrency') $loglevel = hiera('swh::deploy::worker::swh_indexer::fossology_license::loglevel') $task_broker = hiera('swh::deploy::worker::swh_indexer::fossology_license::task_broker') $config_file = '/etc/softwareheritage/indexer/fossology_license.yml' $config = hiera('swh::deploy::worker::swh_indexer::fossology_license::config') - $objstorage_config = hiera('swh::azure_objstorage::config') - $merged_config = merge($config, {'objstorage' => $objstorage_config}) - $task_modules = ['swh.indexer.tasks'] $task_queues = ['swh_indexer_content_fossology_license'] ::profile::swh::deploy::worker::instance {'swh_indexer_fossology_license': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template("<%= @merged_config.to_yaml %>\n"), + content => inline_template("<%= @config.to_yaml %>\n"), } } diff --git a/manifests/swh/deploy/worker/swh_indexer_language.pp b/manifests/swh/deploy/worker/swh_indexer_language.pp index af1a59d3..f22f7be7 100644 --- a/manifests/swh/deploy/worker/swh_indexer_language.pp +++ b/manifests/swh/deploy/worker/swh_indexer_language.pp @@ -1,41 +1,38 @@ # Deployment for swh-indexer-language class profile::swh::deploy::worker::swh_indexer_language { include ::profile::swh::deploy::indexer $concurrency = hiera('swh::deploy::worker::swh_indexer::language::concurrency') $loglevel = hiera('swh::deploy::worker::swh_indexer::language::loglevel') $task_broker = hiera('swh::deploy::worker::swh_indexer::language::task_broker') $config_file = '/etc/softwareheritage/indexer/language.yml' $config = hiera('swh::deploy::worker::swh_indexer::language::config') - $objstorage_config = hiera('swh::azure_objstorage::config') - $merged_config = merge($config, {'objstorage' => $objstorage_config}) - $task_modules = ['swh.indexer.tasks'] $task_queues = ['swh_indexer_content_language'] ::profile::swh::deploy::worker::instance {'swh_indexer_language': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template("<%= @merged_config.to_yaml %>\n"), + content => inline_template("<%= @config.to_yaml %>\n"), } } diff --git a/manifests/swh/deploy/worker/swh_indexer_mimetype.pp b/manifests/swh/deploy/worker/swh_indexer_mimetype.pp index 165cb49d..1c77c6c7 100644 --- a/manifests/swh/deploy/worker/swh_indexer_mimetype.pp +++ b/manifests/swh/deploy/worker/swh_indexer_mimetype.pp @@ -1,41 +1,38 @@ # Deployment for swh-indexer-mimetype class profile::swh::deploy::worker::swh_indexer_mimetype { include ::profile::swh::deploy::indexer $concurrency = hiera('swh::deploy::worker::swh_indexer::mimetype::concurrency') $loglevel = hiera('swh::deploy::worker::swh_indexer::mimetype::loglevel') $task_broker = hiera('swh::deploy::worker::swh_indexer::mimetype::task_broker') $config_file = '/etc/softwareheritage/indexer/mimetype.yml' $config = hiera('swh::deploy::worker::swh_indexer::mimetype::config') - $objstorage_config = hiera('swh::azure_objstorage::config') - $merged_config = merge($config, {'objstorage' => $objstorage_config}) - $task_modules = ['swh.indexer.tasks'] $task_queues = ['swh_indexer_content_mimetype'] ::profile::swh::deploy::worker::instance {'swh_indexer_mimetype': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template("<%= @merged_config.to_yaml %>\n"), + content => inline_template("<%= @config.to_yaml %>\n"), } } diff --git a/manifests/swh/deploy/worker/swh_indexer_rehash.pp b/manifests/swh/deploy/worker/swh_indexer_rehash.pp index 2674e055..3a085d43 100644 --- a/manifests/swh/deploy/worker/swh_indexer_rehash.pp +++ b/manifests/swh/deploy/worker/swh_indexer_rehash.pp @@ -1,41 +1,38 @@ # Deployment for swh-indexer-rehash class profile::swh::deploy::worker::swh_indexer_rehash { include ::profile::swh::deploy::indexer $concurrency = hiera('swh::deploy::worker::swh_indexer::rehash::concurrency') $loglevel = hiera('swh::deploy::worker::swh_indexer::rehash::loglevel') $task_broker = hiera('swh::deploy::worker::swh_indexer::rehash::task_broker') $config_file = '/etc/softwareheritage/indexer/rehash.yml' $config = hiera('swh::deploy::worker::swh_indexer::rehash::config') - $objstorage_config = hiera('swh::azure_objstorage::config') - $merged_config = merge($config, {'objstorage' => $objstorage_config}) - $task_modules = ['swh.indexer.tasks'] $task_queues = ['swh_indexer_content_rehash'] ::profile::swh::deploy::worker::instance {'swh_indexer_rehash': ensure => present, concurrency => $concurrency, loglevel => $loglevel, task_broker => $task_broker, task_modules => $task_modules, task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], File[$config_file], ], } file {$config_file: ensure => 'present', owner => 'swhworker', group => 'swhdev', # Contains passwords mode => '0640', - content => inline_template("<%= @merged_config.to_yaml %>\n"), + content => inline_template("<%= @config.to_yaml %>\n"), } } diff --git a/manifests/swh/deploy/worker/swh_provenance_cache.pp b/manifests/swh/deploy/worker/swh_provenance_cache.pp deleted file mode 100644 index d2e46ec8..00000000 --- a/manifests/swh/deploy/worker/swh_provenance_cache.pp +++ /dev/null @@ -1,45 +0,0 @@ -# Deployment for swh-provenance-cache -class profile::swh::deploy::worker::swh_provenance_cache { - include ::profile::swh::deploy::loader - - $concurrency = hiera('swh::deploy::worker::swh_provenance_cache::concurrency') - $loglevel = hiera('swh::deploy::worker::swh_provenance_cache::loglevel') - $task_broker = hiera('swh::deploy::worker::swh_provenance_cache::task_broker') - - $config_file = '/etc/softwareheritage/storage/provenance_cache.yml' - $config = hiera('swh::deploy::worker::swh_provenance_cache::config') - - $task_modules = ['swh.storage.provenance.tasks'] - $task_queues = [ - 'swh_populate_cache_content_revision', - 'swh_populate_cache_revision_origin' - ] - - $packages = ['python3-swh.storage.provenance'] - - package {$packages: - ensure => 'installed', - } - - ::profile::swh::deploy::worker::instance {'provenance-cache': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Package[$packages], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains password - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/manifests/swh/deploy/worker/swh_reader_git.pp b/manifests/swh/deploy/worker/swh_reader_git.pp deleted file mode 100644 index 4b11a63c..00000000 --- a/manifests/swh/deploy/worker/swh_reader_git.pp +++ /dev/null @@ -1,37 +0,0 @@ -# Deployment for swh-reader-git -class profile::swh::deploy::worker::swh_reader_git { - include ::profile::swh::deploy::base_loader_git - include ::profile::swh::deploy::worker::swh_storage_archiver_azure - - $concurrency = hiera('swh::deploy::worker::swh_reader_git::concurrency') - $loglevel = hiera('swh::deploy::worker::swh_reader_git::loglevel') - $task_broker = hiera('swh::deploy::worker::swh_reader_git::task_broker') - - $config_file = '/etc/softwareheritage/loader/git-remote-reader.yml' - $config = hiera('swh::deploy::worker::swh_reader_git::config') - - $task_modules = ['swh.loader.git.tasks'] - $task_queues = ['swh_reader_git'] - - ::profile::swh::deploy::worker::instance {'swh_reader_git': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Class['profile::swh::deploy::base_loader_git'], - Class['profile::swh::deploy::worker::swh_storage_archiver_azure'], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhworker', - mode => '0644', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/manifests/swh/deploy/worker/swh_storage_archiver_azure.pp b/manifests/swh/deploy/worker/swh_storage_archiver_azure.pp deleted file mode 100644 index d6e8014e..00000000 --- a/manifests/swh/deploy/worker/swh_storage_archiver_azure.pp +++ /dev/null @@ -1,49 +0,0 @@ -# Deployment for swh-storage-archiver-azure -class profile::swh::deploy::worker::swh_storage_archiver_azure { - include ::profile::swh::deploy::archiver - include ::profile::swh::deploy::indexer - - $concurrency = hiera('swh::deploy::worker::swh_storage_archiver_azure::concurrency') - $loglevel = hiera('swh::deploy::worker::swh_storage_archiver_azure::loglevel') - $task_broker = hiera('swh::deploy::worker::swh_storage_archiver_azure::task_broker') - - $config_file = hiera('swh::deploy::worker::swh_storage_archiver_azure::conf_file') - $config = hiera('swh::deploy::worker::swh_storage_archiver_azure::config') - - $storages_config_list = hiera_array( - 'swh::deploy::worker::swh_storage_archiver::storages') - $objstorage_azure_config = hiera('swh::azure_objstorage::config') - - $objstorages_config = $storages_config_list + [ - merge({"host" => "azure"}, $objstorage_azure_config) - ] - - # Create the full configuration - $merged_config = merge($config, { - 'storages' => $objstorages_config - }) - - $task_modules = ['swh.storage.archiver.tasks', 'swh.indexer.tasks'] - $task_queues = ['swh_storage_archive_worker_to_backend'] - - ::profile::swh::deploy::worker::instance {'swh_storage_archiver_azure': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains passwords - mode => '0640', - content => inline_template("<%= @merged_config.to_yaml %>\n"), - } -}