diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -2266,8 +2266,7 @@ # swh::deploy::scheduler::db::password in private data # swh::deploy::scheduler::task_broker::password in private data swh::deploy::scheduler::task_broker: "amqp://swhproducer:%{hiera('swh::deploy::scheduler::task_broker::password')}@rabbitmq:5672/%2f" -swh::deploy::scheduler::listener::log_level: INFO -swh::deploy::scheduler::runner::log_level: INFO +swh::deploy::scheduler::services::log_level: INFO swh::deploy::scheduler::config: <<: *swh_scheduler_local_config celery: @@ -2276,7 +2275,7 @@ - python3-swh.scheduler # subset limited to the save code now tasks for the runner priority -swh::deploy::scheduler::swh-scheduler-runner-priority::config::task-types: +swh::deploy::scheduler::swh-scheduler-runner-priority::config::task_types: - load-git - load-svn - load-archive-files diff --git a/site-modules/profile/manifests/swh/deploy/scheduler.pp b/site-modules/profile/manifests/swh/deploy/scheduler.pp --- a/site-modules/profile/manifests/swh/deploy/scheduler.pp +++ b/site-modules/profile/manifests/swh/deploy/scheduler.pp @@ -6,27 +6,9 @@ $group = lookup('swh::deploy::scheduler::group') $config = lookup('swh::deploy::scheduler::config') - $sentry_dsn = lookup('swh::deploy::scheduler::sentry_dsn', Optional[String], 'first', undef) - $sentry_environment = lookup('swh::deploy::scheduler::sentry_environment', Optional[String], 'first', undef) - $sentry_swh_package = lookup('swh::deploy::scheduler::sentry_swh_package', Optional[String], 'first', undef) - - $listener_log_level = lookup('swh::deploy::scheduler::listener::log_level') - $task_broker = lookup('swh::deploy::scheduler::task_broker') - $listener_service_name = 'swh-scheduler-listener' - $listener_unit_name = "${listener_service_name}.service" - $listener_unit_template = "profile/swh/deploy/scheduler/${listener_service_name}.service.erb" - - $runner_service_name = 'swh-scheduler-runner' - $runner_priority_service_name = 'swh-scheduler-runner-priority' - $packages = lookup('swh::deploy::scheduler::packages') - $services = [ - $listener_service_name, - $runner_service_name, - $runner_priority_service_name, - ] include profile::swh::deploy::base_scheduler @@ -36,55 +18,21 @@ group => $group, mode => '0640', content => inline_template("<%= @config.to_yaml %>\n"), - notify => Service[$services], require => File[$config_dir], } - # Template uses variables - # - $user - # - $group - # - $sentry_{dsn,environment,swh_package} - # - ::systemd::unit_file {$listener_unit_name: - ensure => present, - content => template($listener_unit_template), - notify => Service[$listener_service_name], - } - - service {$listener_service_name: - ensure => running, - enable => true, - require => [ - Package[$packages], - File[$config_file], - Systemd::Unit_File[$listener_unit_name], - ], - subscribe => Package[$packages], - } + # Service definitions + $sentry_dsn = lookup('swh::deploy::scheduler::sentry_dsn', Optional[String], 'first', undef) + $sentry_environment = lookup('swh::deploy::scheduler::sentry_environment', Optional[String], 'first', undef) + $sentry_swh_package = lookup('swh::deploy::scheduler::sentry_swh_package', Optional[String], 'first', undef) - ::profile::swh::deploy::scheduler::runner {$runner_service_name: - user => $user, - group => $group, - packages => $packages, - config_file => $config_file, - sentry_dsn => $sentry_dsn, - sentry_environment => $sentry_environment, - sentry_swh_package => $sentry_swh_package, - } + $services_log_level = lookup('swh::deploy::scheduler::services::log_level') - ::profile::swh::deploy::scheduler::runner {$runner_priority_service_name: - user => $user, - group => $group, - packages => $packages, - config_file => $config_file, - sentry_dsn => $sentry_dsn, - sentry_environment => $sentry_environment, - sentry_swh_package => $sentry_swh_package, - priority => true, - } + include profile::swh::deploy::scheduler::listener + include profile::swh::deploy::scheduler::runner + include profile::swh::deploy::scheduler::runner_priority # scheduler rpc server - ::profile::swh::deploy::rpc_server {'scheduler': config_key => 'scheduler::remote', executable => 'swh.scheduler.api.server:make_app_from_configfile()', diff --git a/site-modules/profile/manifests/swh/deploy/scheduler/listener.pp b/site-modules/profile/manifests/swh/deploy/scheduler/listener.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/scheduler/listener.pp @@ -0,0 +1,7 @@ +# scheduler listener +class profile::swh::deploy::scheduler::listener { + profile::swh::deploy::scheduler::service {"listener": + service_description => 'celery event listener', + service_args => ["start-listener"] + } +} diff --git a/site-modules/profile/manifests/swh/deploy/scheduler/runner.pp b/site-modules/profile/manifests/swh/deploy/scheduler/runner.pp --- a/site-modules/profile/manifests/swh/deploy/scheduler/runner.pp +++ b/site-modules/profile/manifests/swh/deploy/scheduler/runner.pp @@ -1,67 +1,11 @@ -# Instance of scheduler runner -define profile::swh::deploy::scheduler::runner ( - $service_name = $title, - $user = undef, - $group = undef, - $packages = undef, - $config_file = undef, - $priority = false, - $sentry_dsn = undef, - $sentry_environment = undef, - $sentry_swh_package = undef, -) -{ - $runner_log_level = lookup('swh::deploy::scheduler::runner::log_level') +# Runner for old-style tasks - $task_types = lookup( - "swh::deploy::scheduler::${service_name}::config::task-types", - {default_value => []} - ) +class profile::swh::deploy::scheduler::runner { + # The following variable is used by the runner_priority as well + $service_args = ['start-runner', '--period', '10'] - $runner_unit_name = "${service_name}.service" - $runner_unit_template = "profile/swh/deploy/scheduler/swh-scheduler-runner.service.erb" - - $default_command = concat([ - "/usr/bin/swh", - "--log-level ${runner_log_level}", - "scheduler", - "--config-file ${config_file}", - "start-runner", - "--period 10", - ], $priority ? { - true => [ "--with-priority" ], - false => [], - }) - - # complete the command - $runner_command = join( - $task_types.reduce($default_command) | $command, $task_type | { - $command + ["--task-type ${task_type}"] - }, - " " - ) - - # Template uses variables - # - $user - # - $group - # - $sentry_{dsn,environment,swh_package} - # - $runner_command - - ::systemd::unit_file {$runner_unit_name: - ensure => present, - content => template($runner_unit_template), - notify => Service[$service_name], + profile::swh::deploy::scheduler::service {'runner': + service_description => 'runner', + service_args => $service_args, } - - service {$service_name: - ensure => running, - enable => true, - require => [ - Package[$packages], - File[$config_file], - Systemd::Unit_File[$runner_unit_name], - ], - subscribe => Package[$packages], - } - } diff --git a/site-modules/profile/manifests/swh/deploy/scheduler/runner_priority.pp b/site-modules/profile/manifests/swh/deploy/scheduler/runner_priority.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/scheduler/runner_priority.pp @@ -0,0 +1,21 @@ +# Runner for priority tasks +class profile::swh::deploy::scheduler::runner_priority { + $service_name = 'runner-priority' + + $task_types = lookup( + "swh::deploy::scheduler::swh-scheduler-${service_name}::config::task_types", + {default_value => []} + ) + + $priority_service_args = $profile::swh::deploy::scheduler::runner::service_args + ['--with-priority'] + + # add task filter arguments + $service_args = $task_types.reduce($priority_service_args) | $command, $task_type | { + $command + ['--task-type', $task_type] + } + + profile::swh::deploy::scheduler::service {$service_name: + service_description => 'runner for high priority tasks', + service_args => $service_args, + } +} diff --git a/site-modules/profile/manifests/swh/deploy/scheduler/service.pp b/site-modules/profile/manifests/swh/deploy/scheduler/service.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/scheduler/service.pp @@ -0,0 +1,46 @@ +# Instance of scheduler service +define profile::swh::deploy::scheduler::service ( + $service_name = "swh-scheduler-${title}", + $service_description = undef, + $service_args = [], +) +{ + $user = $profile::swh::deploy::scheduler::user + $group = $profile::swh::deploy::scheduler::group + $packages = $profile::swh::deploy::scheduler::packages + $config_file = $profile::swh::deploy::scheduler::config_file + $sentry_dsn = $profile::swh::deploy::scheduler::sentry_dsn + $sentry_environment = $profile::swh::deploy::scheduler::sentry_environment + $sentry_swh_package = $profile::swh::deploy::scheduler::sentry_swh_package + + $services_log_level = $profile::swh::deploy::scheduler::services_log_level + + $unit_name = "${service_name}.service" + $unit_template = "profile/swh/deploy/scheduler/swh-scheduler.service.erb" + + $service_command = join([ + "/usr/bin/swh", + "--log-level ${services_log_level}", + "scheduler", + "--config-file ${config_file}", + ] + $service_args, " ") + + # Template uses variables + # - $user + # - $group + # - $sentry_{dsn,environment,swh_package} + # - $command + ::systemd::unit_file {$unit_name: + ensure => present, + content => template($unit_template), + } + ~> service {$service_name: + ensure => running, + enable => true, + } + + [ + Package[$packages], + File[$config_file], + ] ~> Service[$service_name] +} diff --git a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-listener.service.erb b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-listener.service.erb deleted file mode 100644 --- a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-listener.service.erb +++ /dev/null @@ -1,29 +0,0 @@ -# Scheduler Listener unit file -# Managed by puppet class profile::swh::deploy::scheduler -# Changes will be overwritten - -[Unit] -Description=Software Heritage scheduler listener -Requires=network.target rabbitmq-server.service -After=network.target rabbitmq-server.service - -[Service] -User=<%= @user %> -Group=<%= @group %> -Type=simple -Environment=SWH_LOG_TARGET=journal -<%- if !@sentry_dsn.empty? -%> -Environment=SWH_SENTRY_DSN=<%= @sentry_dsn %> -<%- end -%> -<%- if !@sentry_environment.empty? -%> -Environment=SWH_SENTRY_ENVIRONMENT=<%= @sentry_environment %> -<%- end -%> -<%- if !@sentry_swh_package.empty? -%> -Environment=SWH_MAIN_PACKAGE=<%= @sentry_swh_package %> -<%- end -%> -ExecStart=/usr/bin/swh --log-level <%= @listener_log_level %> scheduler --config-file <%= @config_file %> start-listener -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-runner.service.erb b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler.service.erb rename from site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-runner.service.erb rename to site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler.service.erb --- a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-runner.service.erb +++ b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler.service.erb @@ -1,9 +1,9 @@ -# Scheduler Runner unit file -# Managed by puppet class profile::swh::deploy::scheduler +# Scheduler <%= @service_description %> +# Managed by puppet class profile::swh::deploy::scheduler::service (instance <%= @title %>) # Changes will be overwritten [Unit] -Description=Software Heritage scheduler runner +Description=Software Heritage scheduler <%= @service_description %> Requires=network.target rabbitmq-server.service After=network.target rabbitmq-server.service @@ -21,7 +21,7 @@ <%- if !@sentry_swh_package.empty? -%> Environment=SWH_MAIN_PACKAGE=<%= @sentry_swh_package %> <%- end -%> -ExecStart=<%= @runner_command %> +ExecStart=<%= @service_command %> Restart=always RestartSec=10