diff --git a/data/defaults.yaml b/data/defaults.yaml --- a/data/defaults.yaml +++ b/data/defaults.yaml @@ -1547,7 +1547,6 @@ swh::deploy::worker::swh_loader_deposit::concurrency: 2 swh::deploy::worker::swh_loader_deposit::private_tmp: true swh::deploy::worker::swh_loader_deposit::loglevel: info -swh::deploy::worker::swh_loader_deposit::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" # deposit_basic_auth_swhworker_{username|password} in private_data swh::deploy::worker::swh_loader_deposit::config: storage: "%{alias('swh::remote_service::storage::config')}" @@ -1868,7 +1867,6 @@ swh::deploy::worker::swh_loader_git::config_file: "%{hiera('swh::conf_directory')}/loader_git.yml" swh::deploy::worker::swh_loader_git::concurrency: 2 swh::deploy::worker::swh_loader_git::loglevel: info -swh::deploy::worker::swh_loader_git::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_loader_git::config: storage: "%{alias('swh::remote_service::storage::config')}" save_data: true @@ -1888,7 +1886,6 @@ swh::deploy::worker::swh_loader_debian::config_file: "%{hiera('swh::conf_directory')}/loader_debian.yml" swh::deploy::worker::swh_loader_debian::concurrency: 1 swh::deploy::worker::swh_loader_debian::loglevel: info -swh::deploy::worker::swh_loader_debian::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_loader_debian::config: storage: "%{alias('swh::remote_service::storage::config')}" directory_packet_size: 100 @@ -1905,7 +1902,6 @@ swh::deploy::worker::swh_lister_debian::config_file: "%{hiera('swh::conf_directory')}/lister_debian.yml" swh::deploy::worker::swh_lister_debian::concurrency: 1 swh::deploy::worker::swh_lister_debian::loglevel: info -swh::deploy::worker::swh_lister_debian::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_lister_debian::config: storage: "%{alias('swh::remote_service::storage::config')}" scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" @@ -1925,7 +1921,6 @@ swh::deploy::worker::swh_loader_mercurial::concurrency: 1 swh::deploy::worker::swh_loader_mercurial::private_tmp: true swh::deploy::worker::swh_loader_mercurial::loglevel: info -swh::deploy::worker::swh_loader_mercurial::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_loader_mercurial::config: storage: "%{alias('swh::remote_service::storage::config')}" reduce_effort: False @@ -1943,7 +1938,6 @@ swh::deploy::worker::swh_loader_pypi::concurrency: 1 swh::deploy::worker::swh_loader_pypi::private_tmp: true swh::deploy::worker::swh_loader_pypi::loglevel: info -swh::deploy::worker::swh_loader_pypi::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_loader_pypi::config: storage: "%{alias('swh::remote_service::storage::config')}" celery: @@ -1960,7 +1954,6 @@ swh::deploy::worker::swh_loader_svn::limit_no_file: 8192 swh::deploy::worker::swh_loader_svn::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_loader_svn::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_loader_svn::config: storage: "%{alias('swh::remote_service::storage::config')}" celery: @@ -1978,7 +1971,6 @@ swh::deploy::worker::swh_lister_github::concurrency: 1 swh::deploy::worker::swh_lister_github::loglevel: warning # Contains a password: in private data -swh::deploy::worker::swh_lister_github::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_lister_github::db::host: db swh::deploy::worker::swh_lister_github::db::dbname: lister-github swh::deploy::worker::swh_lister_github::db::user: lister-github @@ -2006,7 +1998,6 @@ swh::deploy::worker::swh_lister_gitlab::concurrency: 1 swh::deploy::worker::swh_lister_gitlab::loglevel: warning # Contains a password: in private data -swh::deploy::worker::swh_lister_gitlab::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_lister_gitlab::db::host: db swh::deploy::worker::swh_lister_gitlab::db::dbname: lister-gitlab swh::deploy::worker::swh_lister_gitlab::db::user: lister-gitlab @@ -2034,7 +2025,6 @@ swh::deploy::worker::swh_lister_pypi::concurrency: 1 swh::deploy::worker::swh_lister_pypi::loglevel: warning # Contains a password: in private data -swh::deploy::worker::swh_lister_pypi::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_lister_pypi::db::host: db swh::deploy::worker::swh_lister_pypi::db::dbname: lister-pypi swh::deploy::worker::swh_lister_pypi::db::user: lister-pypi @@ -2069,7 +2059,6 @@ swh::deploy::worker::swh_storage_archiver::max_tasks_per_child: 500 swh::deploy::worker::swh_storage_archiver::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_storage_archiver::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_storage_archiver::config: archival_max_age: "%{hiera('swh::deploy::worker::swh_storage_archiver::archival_max_age')}" retention_policy: "%{hiera('swh::deploy::worker::swh_storage_archiver::retention_policy')}" @@ -2105,7 +2094,6 @@ swh::deploy::worker::swh_indexer::mimetype::concurrency: 1 swh::deploy::worker::swh_indexer::mimetype::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::mimetype::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_indexer::mimetype::config: scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" @@ -2124,7 +2112,6 @@ swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::concurrency: 2 swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::config: scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" @@ -2137,35 +2124,10 @@ task_queues: - swh.indexer.tasks.OriginMetadata -swh::deploy::worker::swh_indexer::origin_head::config_file: "origin_head.yml" -swh::deploy::worker::swh_indexer::origin_head::concurrency: 1 -swh::deploy::worker::swh_indexer::origin_head::loglevel: info -# Contains a password: in private data -swh::deploy::worker::swh_indexer::origin_head::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_indexer::origin_head::config: - scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" - indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" - storage: "%{alias('swh::remote_service::storage::config::azure')}" - tasks: - revision_metadata: indexer_revision_metadata - origin_intrinsic_metadata: indexer_origin_metadata - -swh::deploy::worker::swh_indexer::revision_metadata::config_file: "revision_metadata.yml" -swh::deploy::worker::swh_indexer::revision_metadata::concurrency: 1 -swh::deploy::worker::swh_indexer::revision_metadata::loglevel: info -# Contains a password: in private data -swh::deploy::worker::swh_indexer::revision_metadata::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_indexer::revision_metadata::config: - scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" - indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" - objstorage: "%{alias('swh::remote_service::objstorage::config')}" - storage: "%{alias('swh::remote_service::storage::config::azure')}" - swh::deploy::worker::swh_indexer::rehash::config_file: "rehash.yml" swh::deploy::worker::swh_indexer::rehash::concurrency: 5 swh::deploy::worker::swh_indexer::rehash::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::rehash::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_indexer::rehash::config: # Needs write access: explicitly configured to use uffizi storage: "%{alias('swh::remote_service::storage::config::uffizi')}" @@ -2179,7 +2141,6 @@ swh::deploy::worker::swh_indexer::fossology_license::concurrency: 1 swh::deploy::worker::swh_indexer::fossology_license::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::fossology_license::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_indexer::fossology_license::config: indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" @@ -2203,7 +2164,6 @@ swh::deploy::worker::swh_indexer::language::concurrency: 3 swh::deploy::worker::swh_indexer::language::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::language::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_indexer::language::config: indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" @@ -2225,7 +2185,6 @@ swh::deploy::worker::swh_indexer::ctags::concurrency: 2 swh::deploy::worker::swh_indexer::ctags::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::ctags::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" # objstorage configuration from swh::azure_objstorage::config is merged in the manifest swh::deploy::worker::swh_indexer::ctags::config: indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" @@ -2648,7 +2607,6 @@ swh::deploy::worker::swh_vault_cooker::config_file: "%{hiera('swh::conf_directory')}/vault_cooker.yml" swh::deploy::worker::swh_vault_cooker::concurrency: 20 swh::deploy::worker::swh_vault_cooker::loglevel: info -swh::deploy::worker::swh_vault_cooker::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" swh::deploy::worker::swh_vault_cooker::conf_file: "%{hiera('swh::conf_directory')}/vault/cooker.yml" swh::deploy::worker::swh_vault_cooker::config: storage: "%{alias('swh::remote_service::storage::config')}" diff --git a/site-modules/profile/manifests/swh/deploy/worker/instance.pp b/site-modules/profile/manifests/swh/deploy/worker/instance.pp --- a/site-modules/profile/manifests/swh/deploy/worker/instance.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/instance.pp @@ -1,10 +1,6 @@ # Instance of a worker define profile::swh::deploy::worker::instance ( $ensure = present, - $task_broker = '', - $task_modules = [], - $task_queues = [], - $task_soft_time_limit = 0, $concurrency = 10, $loglevel = 'info', $max_tasks_per_child = 5, @@ -16,8 +12,6 @@ $service_basename = "swh-worker@${instance_name}" $service_name = "${service_basename}.service" - $config_directory = '/etc/softwareheritage/worker' - $instance_config = "${config_directory}/${instance_name}.ini" case $ensure { 'present', 'running': { @@ -32,32 +26,13 @@ content => template('profile/swh/deploy/worker/parameters.conf.erb'), } - # Uses variables - # - $task_broker - # - $task_modules - # - $task_queues - # - $task_soft_time_limit - file {$instance_config: - ensure => present, - owner => 'swhworker', - group => 'swhdev', - # contains a password for the broker - mode => '0640', - content => template('profile/swh/deploy/worker/instance_config.ini.erb'), - } if $ensure == 'running' { service {$service_basename: ensure => $ensure, - require => [ - File[$instance_config], - ], } } } default: { - file {$instance_config: - ensure => absent, - } ::systemd::dropin_file {"${service_basename}/parameters.conf": ensure => absent, unit => $service_name, diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_ctags.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_ctags.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_ctags.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_ctags.pp @@ -5,14 +5,10 @@ $concurrency = lookup('swh::deploy::worker::swh_indexer::ctags::concurrency') $loglevel = lookup('swh::deploy::worker::swh_indexer::ctags::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::ctags::task_broker') $config_file = lookup('swh::deploy::worker::swh_indexer::ctags::config_file') $config = lookup('swh::deploy::worker::swh_indexer::ctags::config') - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_content_ctags'] - $packages = ['universal-ctags'] package {$packages: ensure => 'present', @@ -22,9 +18,6 @@ ensure => 'stopped', concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp @@ -5,14 +5,10 @@ $concurrency = lookup('swh::deploy::worker::swh_indexer::fossology_license::concurrency') $loglevel = lookup('swh::deploy::worker::swh_indexer::fossology_license::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::fossology_license::task_broker') $config_file = lookup('swh::deploy::worker::swh_indexer::fossology_license::config_file') $config = lookup('swh::deploy::worker::swh_indexer::fossology_license::config') - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_content_fossology_license', 'swh_indexer_content_fossology_license_range'] - $packages = ['fossology-nomossa'] package {$packages: ensure => 'present', @@ -22,9 +18,6 @@ ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_language.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_language.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_language.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_language.pp @@ -5,7 +5,6 @@ $concurrency = lookup('swh::deploy::worker::swh_indexer::language::concurrency') $loglevel = lookup('swh::deploy::worker::swh_indexer::language::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::language::task_broker') $config_file = lookup('swh::deploy::worker::swh_indexer::language::config_file') $config = lookup('swh::deploy::worker::swh_indexer::language::config') @@ -17,9 +16,6 @@ ensure => 'stopped', concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp @@ -5,21 +5,14 @@ $concurrency = lookup('swh::deploy::worker::swh_indexer::mimetype::concurrency') $loglevel = lookup('swh::deploy::worker::swh_indexer::mimetype::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::mimetype::task_broker') $config_file = lookup('swh::deploy::worker::swh_indexer::mimetype::config_file') $config = lookup('swh::deploy::worker::swh_indexer::mimetype::config') - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_content_mimetype', 'swh_indexer_content_mimetype_range'] - Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_mimetype': ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_origin_head.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_origin_head.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_origin_head.pp +++ /dev/null @@ -1,40 +0,0 @@ -# Deployment for swh-indexer-origin-head - -class profile::swh::deploy::worker::swh_indexer_origin_head { - include ::profile::swh::deploy::indexer - - $concurrency = lookup('swh::deploy::worker::swh_indexer::origin_head::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::origin_head::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::origin_head::task_broker') - - $config_file = lookup('swh::deploy::worker::swh_indexer::origin_head::config_file') - $config_directory = lookup('swh::deploy::base_indexer::config_directory') - $config_path = "${config_directory}/${config_file}" - $config = lookup('swh::deploy::worker::swh_indexer::origin_head::config') - - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_origin_head'] - - Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'swh_indexer_origin_head': - ensure => 'stopped', - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Class['profile::swh::deploy::indexer'], - Class['profile::swh::deploy::objstorage_cloud'], - File[$config_path], - ], - } - - file {$config_path: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains passwords - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_origin_intrinsic_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_origin_intrinsic_metadata.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_origin_intrinsic_metadata.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_origin_intrinsic_metadata.pp @@ -5,21 +5,14 @@ $concurrency = lookup('swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::concurrency') $loglevel = lookup('swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::task_broker') $config_file = lookup('swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::config_file') $config = lookup('swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::config') - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_origin_intrinsic_metadata'] - Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_origin_intrinsic_metadata': ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_rehash.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_rehash.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_rehash.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_rehash.pp @@ -5,23 +5,16 @@ $concurrency = lookup('swh::deploy::worker::swh_indexer::rehash::concurrency') $loglevel = lookup('swh::deploy::worker::swh_indexer::rehash::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::rehash::task_broker') $config_file = lookup('swh::deploy::worker::swh_indexer::rehash::config_file') $config_directory = lookup('swh::deploy::base_indexer::config_directory') $config_path = "${config_directory}/${config_file}" $config = lookup('swh::deploy::worker::swh_indexer::rehash::config') - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_content_rehash'] - Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'swh_indexer_rehash': ensure => 'stopped', concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Class['profile::swh::deploy::indexer'], Class['profile::swh::deploy::objstorage_cloud'], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_revision_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_revision_metadata.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_revision_metadata.pp +++ /dev/null @@ -1,40 +0,0 @@ -# Deployment for swh-indexer-revision-metadata - -class profile::swh::deploy::worker::swh_indexer_revision_metadata { - include ::profile::swh::deploy::indexer - - $concurrency = lookup('swh::deploy::worker::swh_indexer::revision_metadata::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::revision_metadata::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::revision_metadata::task_broker') - - $config_file = lookup('swh::deploy::worker::swh_indexer::revision_metadata::config_file') - $config_directory = lookup('swh::deploy::base_indexer::config_directory') - $config_path = "${config_directory}/${config_file}" - $config = lookup('swh::deploy::worker::swh_indexer::revision_metadata::config') - - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_revision_metadata'] - - Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'swh_indexer_revision_metadata': - ensure => 'stopped', - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Class['profile::swh::deploy::indexer'], - Class['profile::swh::deploy::objstorage_cloud'], - File[$config_path], - ], - } - - file {$config_path: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains passwords - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_debian.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_lister_debian.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_debian.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_lister_debian.pp @@ -2,23 +2,16 @@ class profile::swh::deploy::worker::swh_lister_debian { $concurrency = lookup('swh::deploy::worker::swh_lister_debian::concurrency') $loglevel = lookup('swh::deploy::worker::swh_lister_debian::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_lister_debian::task_broker') $config_file = lookup('swh::deploy::worker::swh_lister_debian::config_file') $config = lookup('swh::deploy::worker::swh_lister_debian::config', Hash, 'deep') - $task_modules = ['swh.lister.debian.tasks'] - $task_queues = ['swh_lister_debian'] - include ::profile::swh::deploy::base_lister ::profile::swh::deploy::worker::instance {'lister_debian': ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Package['python3-swh.lister'], File[$config_file], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_github.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_lister_github.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_github.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_lister_github.pp @@ -2,23 +2,16 @@ class profile::swh::deploy::worker::swh_lister_github { $concurrency = lookup('swh::deploy::worker::swh_lister_github::concurrency') $loglevel = lookup('swh::deploy::worker::swh_lister_github::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_lister_github::task_broker') $config_file = lookup('swh::deploy::worker::swh_lister_github::config_file') $config = lookup('swh::deploy::worker::swh_lister_github::config', Hash, 'deep') - $task_modules = ['swh.lister.github.tasks'] - $task_queues = ['swh_lister_github_discover', 'swh_lister_github_refresh'] - include ::profile::swh::deploy::base_lister ::profile::swh::deploy::worker::instance {'lister_github': ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Package['python3-swh.lister'], File[$config_file], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_gitlab.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_lister_gitlab.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_gitlab.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_lister_gitlab.pp @@ -2,23 +2,16 @@ class profile::swh::deploy::worker::swh_lister_gitlab { $concurrency = lookup('swh::deploy::worker::swh_lister_gitlab::concurrency') $loglevel = lookup('swh::deploy::worker::swh_lister_gitlab::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_lister_gitlab::task_broker') $config_file = lookup('swh::deploy::worker::swh_lister_gitlab::config_file') $config = lookup('swh::deploy::worker::swh_lister_gitlab::config', Hash, 'deep') - $task_modules = ['swh.lister.gitlab.tasks'] - $task_queues = ['swh_lister_gitlab_discover', 'swh_lister_gitlab_refresh'] - include ::profile::swh::deploy::base_lister ::profile::swh::deploy::worker::instance {'lister_gitlab': ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Package['python3-swh.lister'], File[$config_file], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_pypi.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_lister_pypi.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_pypi.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_lister_pypi.pp @@ -2,23 +2,16 @@ class profile::swh::deploy::worker::swh_lister_pypi { $concurrency = lookup('swh::deploy::worker::swh_lister_pypi::concurrency') $loglevel = lookup('swh::deploy::worker::swh_lister_pypi::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_lister_pypi::task_broker') $config_file = lookup('swh::deploy::worker::swh_lister_pypi::config_file') $config = lookup('swh::deploy::worker::swh_lister_pypi::config', Hash, 'deep') - $task_modules = ['swh.lister.pypi.tasks'] - $task_queues = ['swh_lister_pypi_refresh'] - include ::profile::swh::deploy::base_lister ::profile::swh::deploy::worker::instance {'lister_pypi': ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Package['python3-swh.lister'], File[$config_file], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_debian.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_debian.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_debian.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_debian.pp @@ -2,14 +2,10 @@ class profile::swh::deploy::worker::swh_loader_debian { $concurrency = lookup('swh::deploy::worker::swh_loader_debian::concurrency') $loglevel = lookup('swh::deploy::worker::swh_loader_debian::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_debian::task_broker') $config_file = lookup('swh::deploy::worker::swh_loader_debian::config_file') $config = lookup('swh::deploy::worker::swh_loader_debian::config') - $task_modules = ['swh.loader.debian.tasks'] - $task_queues = ['swh_loader_debian'] - $packages = ['python3-swh.loader.debian'] package {$packages: @@ -20,9 +16,6 @@ ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Package[$packages], File[$config_file], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_deposit.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_deposit.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_deposit.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_deposit.pp @@ -2,14 +2,10 @@ class profile::swh::deploy::worker::swh_loader_deposit { $concurrency = lookup('swh::deploy::worker::swh_loader_deposit::concurrency') $loglevel = lookup('swh::deploy::worker::swh_loader_deposit::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_deposit::task_broker') $config_file = lookup('swh::deploy::worker::swh_loader_deposit::config_file') $config = lookup('swh::deploy::worker::swh_loader_deposit::config') - $task_modules = ['swh.deposit.loader.tasks'] - $task_queues = ['swh_checker_deposit', 'swh_loader_deposit'] - $packages = ['python3-swh.deposit.loader'] $private_tmp = lookup('swh::deploy::worker::swh_loader_deposit::private_tmp') @@ -25,9 +21,6 @@ ensure => running, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, private_tmp => $private_tmp, require => [ Package[$packages], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_git.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_git.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_git.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_git.pp @@ -4,21 +4,14 @@ $concurrency = lookup('swh::deploy::worker::swh_loader_git::concurrency') $loglevel = lookup('swh::deploy::worker::swh_loader_git::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_git::task_broker') $config_file = lookup('swh::deploy::worker::swh_loader_git::config_file') $config = lookup('swh::deploy::worker::swh_loader_git::config') - $task_modules = ['swh.loader.git.tasks'] - $task_queues = ['swh_loader_git'] - ::profile::swh::deploy::worker::instance {'loader_git': ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Class['profile::swh::deploy::base_loader_git'], File[$config_file], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_mercurial.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_mercurial.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_mercurial.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_mercurial.pp @@ -4,14 +4,10 @@ $concurrency = lookup('swh::deploy::worker::swh_loader_mercurial::concurrency') $loglevel = lookup('swh::deploy::worker::swh_loader_mercurial::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_mercurial::task_broker') $config_file = lookup('swh::deploy::worker::swh_loader_mercurial::config_file') $config = lookup('swh::deploy::worker::swh_loader_mercurial::config') - $task_modules = ['swh.loader.mercurial.tasks'] - $task_queues = ['swh_loader_mercurial', 'swh_loader_mercurial_archive'] - $service_name = 'loader_mercurial' $private_tmp = lookup('swh::deploy::worker::swh_loader_mercurial::private_tmp') @@ -26,9 +22,6 @@ ensure => running, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, private_tmp => $private_tmp, require => [ File[$config_file], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_pypi.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_pypi.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_pypi.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_pypi.pp @@ -2,13 +2,10 @@ class profile::swh::deploy::worker::swh_loader_pypi { $concurrency = lookup('swh::deploy::worker::swh_loader_pypi::concurrency') $loglevel = lookup('swh::deploy::worker::swh_loader_pypi::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_pypi::task_broker') $config_file = lookup('swh::deploy::worker::swh_loader_pypi::config_file') $config = lookup('swh::deploy::worker::swh_loader_pypi::config') - $task_modules = ['swh.loader.pypi.tasks'] - $task_queues = ['swh_loader_pypi'] $private_tmp = lookup('swh::deploy::worker::swh_loader_pypi::private_tmp') $packages = ['python3-swh.loader.pypi'] @@ -21,9 +18,6 @@ ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, private_tmp => $private_tmp, require => [ Package[$packages], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_svn.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_svn.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_svn.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_svn.pp @@ -2,14 +2,10 @@ class profile::swh::deploy::worker::swh_loader_svn { $concurrency = lookup('swh::deploy::worker::swh_loader_svn::concurrency') $loglevel = lookup('swh::deploy::worker::swh_loader_svn::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_svn::task_broker') $config_file = '/etc/softwareheritage/loader/svn.yml' $config = lookup('swh::deploy::worker::swh_loader_svn::config') - $task_modules = ['swh.loader.svn.tasks'] - $task_queues = ['swh_loader_svn', 'swh_loader_svn_mount_and_load'] - $packages = ['python3-swh.loader.svn'] $limit_no_file = lookup('swh::deploy::worker::swh_loader_svn::limit_no_file') $private_tmp = lookup('swh::deploy::worker::swh_loader_svn::private_tmp') @@ -22,9 +18,6 @@ ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, limit_no_file => $limit_no_file, private_tmp => $private_tmp, require => [ diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_storage_archiver.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_storage_archiver.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_storage_archiver.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_storage_archiver.pp @@ -5,21 +5,14 @@ $concurrency = lookup('swh::deploy::worker::swh_storage_archiver::concurrency') $max_tasks_per_child = lookup('swh::deploy::worker::swh_storage_archiver::max_tasks_per_child') $loglevel = lookup('swh::deploy::worker::swh_storage_archiver::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_storage_archiver::task_broker') $config_file = lookup('swh::deploy::worker::swh_storage_archiver::conf_file') $config = lookup('swh::deploy::worker::swh_storage_archiver::config') - $task_modules = ['swh.archiver.tasks'] - $task_queues = ['swh_storage_archive_worker'] - ::profile::swh::deploy::worker::instance {'swh_storage_archiver': ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, max_tasks_per_child => $max_tasks_per_child, require => [ File[$config_file], diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_vault_cooker.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_vault_cooker.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_vault_cooker.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/swh_vault_cooker.pp @@ -5,21 +5,14 @@ $concurrency = lookup('swh::deploy::worker::swh_vault_cooker::concurrency') $loglevel = lookup('swh::deploy::worker::swh_vault_cooker::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_vault_cooker::task_broker') $conf_file = lookup('swh::deploy::worker::swh_vault_cooker::config_file') $config = lookup('swh::deploy::worker::swh_vault_cooker::config') - $task_modules = ['swh.vault.cooking_tasks'] - $task_queues = ['swh_vault_cooking', 'swh_vault_batch_cooking'] - ::profile::swh::deploy::worker::instance {'vault_cooker': ensure => present, concurrency => $concurrency, loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, require => [ Package[$packages], File[$conf_file], diff --git a/site-modules/profile/templates/swh/deploy/worker/instance_config.ini.erb b/site-modules/profile/templates/swh/deploy/worker/instance_config.ini.erb deleted file mode 100644 --- a/site-modules/profile/templates/swh/deploy/worker/instance_config.ini.erb +++ /dev/null @@ -1,8 +0,0 @@ -# Managed by puppet - modifications will be overwritten -# In defined class profile::swh::deploy::worker::instance - -[main] -task_broker = <%= @task_broker %> -task_modules = <%= @task_modules.join(', ') %> -task_queues = <%= @task_queues.join(', ') %> -task_soft_time_limit = <%= @task_soft_time_limit %>