diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -2453,6 +2453,7 @@ swh::deploy::worker::loader_git::max_tasks_per_child: 100 swh::deploy::worker::loader_git::temp_file_cutoff: 536870912 # 512 * 1024 * 1024 swh::deploy::worker::loader_git::loglevel: info +swh::deploy::worker::loader_git::load_metadata: false swh::deploy::loader_git::queues: - swh.loader.git.tasks.UpdateGitRepository diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -132,6 +132,9 @@ # do not save pack swh::deploy::worker::loader_git::save_data_path: "" swh::deploy::worker::loader_git::concurrency: 1 +swh::deploy::worker::loader_git::load_metadata: true +swh::deploy::worker::loader_git::extra_config: + credentials: "%{alias('swh::deploy::worker::lister::config::credentials')}" zookeeper::clusters: rocquencourt_staging: diff --git a/site-modules/profile/manifests/swh/deploy/worker/instance.pp b/site-modules/profile/manifests/swh/deploy/worker/instance.pp --- a/site-modules/profile/manifests/swh/deploy/worker/instance.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/instance.pp @@ -6,6 +6,9 @@ # False, the default, for other workers whose recurring tasks are scheduled with # next-gen scheduler-runner. Their status are updated through a journal client so no # need for the events. +# @param extra_config +# Extra configuration dict to merge into the default $config loaded for the service. +# Typically, that'd be needed to provide extra sensible information like credentials. define profile::swh::deploy::worker::instance ( $ensure = present, $instance_name = $title, @@ -14,6 +17,7 @@ $private_tmp = undef, $merge_policy = 'deep', $send_task_events = false, + Hash[String, Any] $extra_config = {}, ) { include ::profile::swh::deploy::worker::base @@ -31,7 +35,11 @@ $max_tasks_per_child = lookup("swh::deploy::worker::${instance_name}::max_tasks_per_child", Integer, first, 5) $loglevel = lookup("swh::deploy::worker::${instance_name}::loglevel") $config_file = lookup("swh::deploy::worker::${instance_name}::config_file") - $config = lookup("swh::deploy::worker::${instance_name}::config", Hash, $merge_policy) + # Merge the default $config with $extra_config (if any) + $config = deep_merge( + lookup("swh::deploy::worker::${instance_name}::config", Hash, $merge_policy), + $extra_config + ) $sentry_dsn = lookup("swh::deploy::${sentry_name}::sentry_dsn", Optional[String], 'first', undef) $sentry_environment = lookup("swh::deploy::${sentry_name}::sentry_environment", Optional[String], 'first', undef) diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp --- a/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp @@ -2,10 +2,24 @@ class profile::swh::deploy::worker::loader_git { include ::profile::swh::deploy::base_loader_git + $load_metadata = lookup('swh::deploy::worker::loader_git::load_metadata') + $packages = [$::profile::swh::deploy::base_loader_git::packages] + + if $load_metadata { + $extra_config = lookup('swh::deploy::worker::loader_git::extra_config', { + default_value => {} + }) + $extra_packages = ['python3-swh.loader.metadata'] + ensure_packages($extra_packages) + $all_packages = $packages + $extra_packages + } else { + $extra_config = {} + $all_packages = $packages + } + ::profile::swh::deploy::worker::instance {'loader_git': - ensure => present, - require => [ - Package[$::profile::swh::deploy::base_loader_git::packages], - ], + ensure => present, + require => Package[$all_packages], + extra_config => $extra_config } }