Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9343752
D6439.id23390.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
9 KB
Subscribers
None
D6439.id23390.diff
View Options
diff --git a/site-modules/profile/manifests/swh/deploy/worker/base.pp b/site-modules/profile/manifests/swh/deploy/worker/base.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/base.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/base.pp
@@ -2,6 +2,7 @@
class profile::swh::deploy::worker::base {
$systemd_template_unit_name = 'swh-worker@.service'
+ $systemd_template_events_unit_name = 'swh-worker-events@.service'
$systemd_unit_name = 'swh-worker.service'
$systemd_slice_name = 'system-swh\x2dworker.slice'
@@ -11,11 +12,23 @@
ensure => installed,
}
+ # Parametrize the celery worker to actually send task events if required
+ if $send_task_events {
+ $extra_command_flag = "--events"
+ } else {
+ $extra_command_flag = ""
+ }
+
::systemd::unit_file {$systemd_template_unit_name:
ensure => 'present',
content => template("profile/swh/deploy/worker/${systemd_template_unit_name}.erb"),
}
+ ::systemd::unit_file {$systemd_template_events_unit_name:
+ ensure => 'present',
+ content => template("profile/swh/deploy/worker/${systemd_template_events_unit_name}.erb"),
+ }
+
::systemd::unit_file {$systemd_unit_name:
ensure => 'present',
source => "puppet:///modules/profile/swh/deploy/worker/${systemd_unit_name}",
diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
@@ -4,9 +4,10 @@
include ::profile::swh::deploy::indexer
Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_mimetype':
- ensure => present,
- sentry_name => 'indexer',
- require => [
+ ensure => present,
+ sentry_name => 'indexer',
+ send_task_events => true,
+ require => [
Class['profile::swh::deploy::indexer']
],
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
@@ -10,9 +10,10 @@
}
Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_fossology_license':
- ensure => present,
- sentry_name => 'indexer',
- require => [
+ ensure => present,
+ sentry_name => 'indexer',
+ send_task_events => true,
+ require => [
Class['profile::swh::deploy::indexer'],
Package[$packages],
],
diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
@@ -4,9 +4,10 @@
include ::profile::swh::deploy::indexer
Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_origin_intrinsic_metadata':
- ensure => present,
- sentry_name => 'indexer',
- require => [
+ ensure => present,
+ sentry_name => 'indexer',
+ send_task_events => true,
+ require => [
Class['profile::swh::deploy::indexer'],
],
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/instance.pp b/site-modules/profile/manifests/swh/deploy/worker/instance.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/instance.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/instance.pp
@@ -1,16 +1,24 @@
# Instance of a worker
define profile::swh::deploy::worker::instance (
- $ensure = present,
- $instance_name = $title,
- $sentry_name = $title,
- $limit_no_file = undef,
- $private_tmp = undef,
- $merge_policy = 'deep',
+ $ensure = present,
+ $instance_name = $title,
+ $sentry_name = $title,
+ $limit_no_file = undef,
+ $private_tmp = undef,
+ $merge_policy = 'deep',
+ $send_task_events = false,
)
{
include ::profile::swh::deploy::worker::base
- $service_basename = "swh-worker@${instance_name}"
+ if $send_task_events {
+ $prefix_unit_name = "swh-worker-events" # traditional workers sends events
+ } else {
+ $prefix_unit_name = "swh-worker" # recurring workers no longer send events
+ }
+
+ $service_basename = "${prefix_unit_name}@${instance_name}"
+ $unit_file_target = "/etc/systemd/system/${prefix_unit_name}@.service"
$service_name = "${service_basename}.service"
$concurrency = lookup("swh::deploy::worker::${instance_name}::concurrency")
$max_tasks_per_child = lookup("swh::deploy::worker::${instance_name}::max_tasks_per_child", Integer, first, 5)
@@ -53,12 +61,19 @@
$service_ensure = undef
}
+ ::systemd::unit_file {$service_name:
+ ensure => $service_ensure,
+ enable => true,
+ target => $unit_file_target,
+ }
+
service {$service_basename:
ensure => $service_ensure,
enable => true,
require => [
File[$config_file],
- ]
+ Systemd::Unit_file[$service_name],
+ ],
}
profile::cron::d {"swh-worker-${instance_name}-autorestart":
diff --git a/site-modules/profile/manifests/swh/deploy/worker/lister.pp b/site-modules/profile/manifests/swh/deploy/worker/lister.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/lister.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/lister.pp
@@ -7,11 +7,12 @@
}
::profile::swh::deploy::worker::instance {'lister':
- ensure => present,
- require => [
+ ensure => present,
+ send_task_events => true,
+ require => [
Package['python3-swh.lister'],
],
- merge_policy => 'first', # do not merge configuration, take the first
- # encountered configuration
+ merge_policy => 'first', # do not merge configuration, take the first
+ # encountered configuration
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp
@@ -7,8 +7,9 @@
$packages = $::profile::swh::deploy::base_loader_git::packages + $::profile::swh::deploy::base_loader_mercurial::packages + $::profile::swh::deploy::base_loader_svn::packages
::profile::swh::deploy::worker::instance {'loader_high_priority':
- ensure => present,
- require => Package[$packages],
+ ensure => present,
+ send_task_events => true,
+ require => Package[$packages],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_oneshot.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_oneshot.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/loader_oneshot.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/loader_oneshot.pp
@@ -7,8 +7,8 @@
$packages = $::profile::swh::deploy::base_loader_git::packages + $::profile::swh::deploy::base_loader_mercurial::packages + $::profile::swh::deploy::base_loader_svn::packages
::profile::swh::deploy::worker::instance {'loader_oneshot':
- ensure => present,
- require => Package[$packages],
+ ensure => present,
+ require => Package[$packages],
}
}
diff --git a/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp b/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp
--- a/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp
+++ b/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp
@@ -15,9 +15,10 @@
}
::profile::swh::deploy::worker::instance {$instance_name:
- ensure => present,
- sentry_name => 'vault',
- require => [
+ ensure => present,
+ sentry_name => 'vault',
+ send_task_events => true,
+ require => [
Package[$extra_packages],
Package[$::profile::swh::deploy::base_vault::packages],
],
diff --git a/site-modules/profile/templates/swh/deploy/worker/swh-worker-events@.service.erb b/site-modules/profile/templates/swh/deploy/worker/swh-worker-events@.service.erb
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/templates/swh/deploy/worker/swh-worker-events@.service.erb
@@ -0,0 +1,28 @@
+[Unit]
+Description=Software Heritage Worker (%i)
+After=network.target
+
+[Service]
+User=swhworker
+Group=swhworker
+
+Type=simple
+Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/%i.yml
+Environment=SWH_LOG_TARGET=journal
+Environment=CONCURRENCY=10
+Environment=MAX_TASKS_PER_CHILD=5
+Environment=LOGLEVEL=info
+Environment=CELERY_HOSTNAME=<%= @celery_hostname %>
+ExecStart=/usr/bin/python3 -m celery worker -n %i@${CELERY_HOSTNAME} --app=swh.scheduler.celery_backend.config.app --pool=prefork --concurrency=${CONCURRENCY} --maxtasksperchild=${MAX_TASKS_PER_CHILD} -Ofair --loglevel=${LOGLEVEL} --without-gossip --without-mingle --without-heartbeat
+
+KillMode=process
+KillSignal=SIGTERM
+TimeoutStopSec=15m
+
+OOMPolicy=kill
+
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Thu, Jul 3, 1:48 PM (5 d, 6 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3225909
Attached To
D6439: Make workers send task events only when required
Event Timeline
Log In to Comment