diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp --- a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp @@ -4,9 +4,10 @@ include ::profile::swh::deploy::indexer Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_mimetype': - ensure => present, - sentry_name => 'indexer', - require => [ + ensure => present, + sentry_name => 'indexer', + send_task_events => true, + require => [ Class['profile::swh::deploy::indexer'] ], } diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp --- a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp @@ -10,9 +10,10 @@ } Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_fossology_license': - ensure => present, - sentry_name => 'indexer', - require => [ + ensure => present, + sentry_name => 'indexer', + send_task_events => true, + require => [ Class['profile::swh::deploy::indexer'], Package[$packages], ], diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp --- a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp @@ -4,9 +4,10 @@ include ::profile::swh::deploy::indexer Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_origin_intrinsic_metadata': - ensure => present, - sentry_name => 'indexer', - require => [ + ensure => present, + sentry_name => 'indexer', + send_task_events => true, + require => [ Class['profile::swh::deploy::indexer'], ], } diff --git a/site-modules/profile/manifests/swh/deploy/worker/instance.pp b/site-modules/profile/manifests/swh/deploy/worker/instance.pp --- a/site-modules/profile/manifests/swh/deploy/worker/instance.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/instance.pp @@ -1,4 +1,11 @@ # Instance of a worker + +# @param send_task_events +# True for workers whose tasks are still scheduled with scheduler-runner (they need to +# report their status to the listener which updates the scheduler accordingly). +# False, the default, for other workers whose recurring tasks are scheduled with +# next-gen scheduler-runner. Their status are updated through a journal client so no +# need for the events. define profile::swh::deploy::worker::instance ( $ensure = present, $instance_name = $title, @@ -6,10 +13,18 @@ $limit_no_file = undef, $private_tmp = undef, $merge_policy = 'deep', + $send_task_events = false, ) { include ::profile::swh::deploy::worker::base + # Parametrize the celery worker to actually send task events if required + if $send_task_events { + $celery_worker_extra_args = "--events" + } else { + $celery_worker_extra_args = "" + } + $service_basename = "swh-worker@${instance_name}" $service_name = "${service_basename}.service" $concurrency = lookup("swh::deploy::worker::${instance_name}::concurrency") diff --git a/site-modules/profile/manifests/swh/deploy/worker/lister.pp b/site-modules/profile/manifests/swh/deploy/worker/lister.pp --- a/site-modules/profile/manifests/swh/deploy/worker/lister.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/lister.pp @@ -7,11 +7,12 @@ } ::profile::swh::deploy::worker::instance {'lister': - ensure => present, - require => [ + ensure => present, + send_task_events => true, + require => [ Package['python3-swh.lister'], ], - merge_policy => 'first', # do not merge configuration, take the first - # encountered configuration + merge_policy => 'first', # do not merge configuration, take the first + # encountered configuration } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp --- a/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp @@ -7,8 +7,9 @@ $packages = $::profile::swh::deploy::base_loader_git::packages + $::profile::swh::deploy::base_loader_mercurial::packages + $::profile::swh::deploy::base_loader_svn::packages ::profile::swh::deploy::worker::instance {'loader_high_priority': - ensure => present, - require => Package[$packages], + ensure => present, + send_task_events => true, + require => Package[$packages], } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp b/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp --- a/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp @@ -15,9 +15,10 @@ } ::profile::swh::deploy::worker::instance {$instance_name: - ensure => present, - sentry_name => 'vault', - require => [ + ensure => present, + sentry_name => 'vault', + send_task_events => true, + require => [ Package[$extra_packages], Package[$::profile::swh::deploy::base_vault::packages], ], diff --git a/site-modules/profile/templates/swh/deploy/worker/parameters.conf.erb b/site-modules/profile/templates/swh/deploy/worker/parameters.conf.erb --- a/site-modules/profile/templates/swh/deploy/worker/parameters.conf.erb +++ b/site-modules/profile/templates/swh/deploy/worker/parameters.conf.erb @@ -20,3 +20,6 @@ <% if @private_tmp %> PrivateTmp=<%= @private_tmp %> <% end %> +<%- if !@celery_worker_extra_args.nil? and !@celery_worker_extra_args.empty? -%> +Environment=CELERY_WORKER_EXTRA_ARGS=<%= @celery_worker_extra_args %> +<%- end -%> diff --git a/site-modules/profile/templates/swh/deploy/worker/swh-worker@.service.erb b/site-modules/profile/templates/swh/deploy/worker/swh-worker@.service.erb --- a/site-modules/profile/templates/swh/deploy/worker/swh-worker@.service.erb +++ b/site-modules/profile/templates/swh/deploy/worker/swh-worker@.service.erb @@ -1,3 +1,6 @@ +# Managed by puppet - modifications will be overwritten +# In defined class profile::swh::deploy::worker::base + [Unit] Description=Software Heritage Worker (%i) After=network.target @@ -7,13 +10,16 @@ Group=swhworker Type=simple +# Following environment variables can be overriden in the respective +# swh-worker@.service.d/parameters.conf Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/%i.yml Environment=SWH_LOG_TARGET=journal Environment=CONCURRENCY=10 Environment=MAX_TASKS_PER_CHILD=5 Environment=LOGLEVEL=info Environment=CELERY_HOSTNAME=<%= @celery_hostname %> -ExecStart=/usr/bin/python3 -m celery worker -n %i@${CELERY_HOSTNAME} --app=swh.scheduler.celery_backend.config.app --pool=prefork --events --concurrency=${CONCURRENCY} --maxtasksperchild=${MAX_TASKS_PER_CHILD} -Ofair --loglevel=${LOGLEVEL} --without-gossip --without-mingle --without-heartbeat +Environment=CELERY_WORKER_EXTRA_ARGS= +ExecStart=/usr/bin/python3 -m celery worker -n %i@${CELERY_HOSTNAME} --app=swh.scheduler.celery_backend.config.app --pool=prefork ${CELERY_WORKER_EXTRA_ARGS} --concurrency=${CONCURRENCY} --maxtasksperchild=${MAX_TASKS_PER_CHILD} -Ofair --loglevel=${LOGLEVEL} --without-gossip --without-mingle --without-heartbeat KillMode=process KillSignal=SIGTERM