Page MenuHomeSoftware Heritage
Files F9348179

D6439.diff
No OneTemporary
Actions

D6439.diff
View Options

diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
@@ -4,9 +4,10 @@
include ::profile::swh::deploy::indexer
Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_mimetype':
- ensure => present,
- sentry_name => 'indexer',
- require => [
+ ensure => present,
+ sentry_name => 'indexer',
+ send_task_events => true,
+ require => [
Class['profile::swh::deploy::indexer']
],
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
@@ -10,9 +10,10 @@
}
Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_fossology_license':
- ensure => present,
- sentry_name => 'indexer',
- require => [
+ ensure => present,
+ sentry_name => 'indexer',
+ send_task_events => true,
+ require => [
Class['profile::swh::deploy::indexer'],
Package[$packages],
],
diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
@@ -4,9 +4,10 @@
include ::profile::swh::deploy::indexer
Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_origin_intrinsic_metadata':
- ensure => present,
- sentry_name => 'indexer',
- require => [
+ ensure => present,
+ sentry_name => 'indexer',
+ send_task_events => true,
+ require => [
Class['profile::swh::deploy::indexer'],
],
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/instance.pp b/site-modules/profile/manifests/swh/deploy/worker/instance.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/instance.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/instance.pp
@@ -1,4 +1,11 @@
# Instance of a worker
+
+# @param send_task_events
+# True for workers whose tasks are still scheduled with scheduler-runner (they need to
+# report their status to the listener which updates the scheduler accordingly).
+# False, the default, for other workers whose recurring tasks are scheduled with
+# next-gen scheduler-runner. Their status are updated through a journal client so no
+# need for the events.
define profile::swh::deploy::worker::instance (
$ensure = present,
$instance_name = $title,
@@ -6,10 +13,18 @@
$limit_no_file = undef,
$private_tmp = undef,
$merge_policy = 'deep',
+ $send_task_events = false,
)
{
include ::profile::swh::deploy::worker::base
+ # Parametrize the celery worker to actually send task events if required
+ if $send_task_events {
+ $celery_worker_extra_args = "--events"
+ } else {
+ $celery_worker_extra_args = ""
+ }
+
$service_basename = "swh-worker@${instance_name}"
$service_name = "${service_basename}.service"
$concurrency = lookup("swh::deploy::worker::${instance_name}::concurrency")
diff --git a/site-modules/profile/manifests/swh/deploy/worker/lister.pp b/site-modules/profile/manifests/swh/deploy/worker/lister.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/lister.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/lister.pp
@@ -7,11 +7,12 @@
}
::profile::swh::deploy::worker::instance {'lister':
- ensure => present,
- require => [
+ ensure => present,
+ send_task_events => true,
+ require => [
Package['python3-swh.lister'],
],
- merge_policy => 'first', # do not merge configuration, take the first
- # encountered configuration
+ merge_policy => 'first', # do not merge configuration, take the first
+ # encountered configuration
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp
@@ -7,8 +7,9 @@
$packages = $::profile::swh::deploy::base_loader_git::packages + $::profile::swh::deploy::base_loader_mercurial::packages + $::profile::swh::deploy::base_loader_svn::packages
::profile::swh::deploy::worker::instance {'loader_high_priority':
- ensure => present,
- require => Package[$packages],
+ ensure => present,
+ send_task_events => true,
+ require => Package[$packages],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp b/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp
@@ -15,9 +15,10 @@
}
::profile::swh::deploy::worker::instance {$instance_name:
- ensure => present,
- sentry_name => 'vault',
- require => [
+ ensure => present,
+ sentry_name => 'vault',
+ send_task_events => true,
+ require => [
Package[$extra_packages],
Package[$::profile::swh::deploy::base_vault::packages],
],
diff --git a/site-modules/profile/templates/swh/deploy/worker/parameters.conf.erb b/site-modules/profile/templates/swh/deploy/worker/parameters.conf.erb
--- a/site-modules/profile/templates/swh/deploy/worker/parameters.conf.erb
+++ b/site-modules/profile/templates/swh/deploy/worker/parameters.conf.erb
@@ -20,3 +20,6 @@
<% if @private_tmp %>
PrivateTmp=<%= @private_tmp %>
<% end %>
+<%- if !@celery_worker_extra_args.nil? and !@celery_worker_extra_args.empty? -%>
+Environment=CELERY_WORKER_EXTRA_ARGS=<%= @celery_worker_extra_args %>
+<%- end -%>
diff --git a/site-modules/profile/templates/swh/deploy/worker/swh-worker@.service.erb b/site-modules/profile/templates/swh/deploy/worker/swh-worker@.service.erb
--- a/site-modules/profile/templates/swh/deploy/worker/swh-worker@.service.erb
+++ b/site-modules/profile/templates/swh/deploy/worker/swh-worker@.service.erb
@@ -1,3 +1,6 @@
+# Managed by puppet - modifications will be overwritten
+# In defined class profile::swh::deploy::worker::base
+
[Unit]
Description=Software Heritage Worker (%i)
After=network.target
@@ -7,13 +10,16 @@
Group=swhworker
Type=simple
+# Following environment variables can be overriden in the respective
+# swh-worker@<service>.service.d/parameters.conf
Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/%i.yml
Environment=SWH_LOG_TARGET=journal
Environment=CONCURRENCY=10
Environment=MAX_TASKS_PER_CHILD=5
Environment=LOGLEVEL=info
Environment=CELERY_HOSTNAME=<%= @celery_hostname %>
-ExecStart=/usr/bin/python3 -m celery worker -n %i@${CELERY_HOSTNAME} --app=swh.scheduler.celery_backend.config.app --pool=prefork --events --concurrency=${CONCURRENCY} --maxtasksperchild=${MAX_TASKS_PER_CHILD} -Ofair --loglevel=${LOGLEVEL} --without-gossip --without-mingle --without-heartbeat
+Environment=CELERY_WORKER_EXTRA_ARGS=
+ExecStart=/usr/bin/python3 -m celery worker -n %i@${CELERY_HOSTNAME} --app=swh.scheduler.celery_backend.config.app --pool=prefork ${CELERY_WORKER_EXTRA_ARGS} --concurrency=${CONCURRENCY} --maxtasksperchild=${MAX_TASKS_PER_CHILD} -Ofair --loglevel=${LOGLEVEL} --without-gossip --without-mingle --without-heartbeat
KillMode=process
KillSignal=SIGTERM

File Metadata

Mime Type
text/plain
Expires
Jul 3 2025, 6:15 PM (5 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3220671

Event Timeline

About · Developer information · Legal