diff --git a/Puppetfile b/Puppetfile index a8a326ae..64810a93 100644 --- a/Puppetfile +++ b/Puppetfile @@ -1,208 +1,208 @@ mod 'dar', :git => 'https://forge.softwareheritage.org/source/puppet-swh-dar', :branch => :control_branch, :default_branch => 'master' mod 'gunicorn', :git => 'https://forge.softwareheritage.org/source/puppet-swh-gunicorn', :branch => :control_branch, :default_branch => 'master' mod 'mediawiki', :git => 'https://forge.softwareheritage.org/source/puppet-swh-mediawiki', :branch => :control_branch, :default_branch => 'master' mod 'postfix', :git => 'https://forge.softwareheritage.org/source/puppet-swh-postfix', :branch => :control_branch, :default_branch => 'master' mod 'uwsgi', :git => 'https://forge.softwareheritage.org/source/puppet-swh-uwsgi', :branch => :control_branch, :default_branch => 'master' mod 'apache', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-apache', :tag => 'v5.3.0' mod 'apt', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-apt', :tag => 'v7.3.0' mod 'archive', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-archive', :tag => 'v4.4.0' mod 'bind', :git => 'https://forge.softwareheritage.org/source/puppet-inkblot-bind', :ref => '7.4.0' mod 'cassandra', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-cassandra', :ref => 'master' mod 'ceph', :git => 'https://forge.softwareheritage.org/source/puppet-openstack-ceph', :ref => 'master' mod 'concat', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-concat', :tag => 'v6.1.0' mod 'cups', :git => 'https://forge.softwareheritage.org/source/puppet-mosen-cups', :ref => 'master' mod 'datacat', :git => 'https://forge.softwareheritage.org/source/puppet-richardc-datacat', :ref => '0.6.2' mod 'debconf', :git => 'https://forge.softwareheritage.org/source/puppet-stm-debconf', :ref => 'v3.0.0' mod 'debnet', :git => 'https://forge.softwareheritage.org/source/puppet-trepasi-debnet', :ref => '8d856df078352a8848a43ca0ee9f2ef9086b343a' mod 'docker', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-docker', :ref => 'v3.9.0' mod 'elasticsearch', :git => 'https://forge.softwareheritage.org/source/puppet-elastic-elasticsearch', :ref => '6.4.0' mod 'extlib', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-extlib', :tag => 'v4.1.0' mod 'grafana', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-grafana', :tag => 'v6.1.0' mod 'hitch', :git => 'https://forge.softwareheritage.org/source/puppet-ssm-hitch', :ref => 'feature/additional-config-0.1.5' mod 'icinga2', :git => 'https://forge.softwareheritage.org/source/puppet-icinga-icinga2', :tag => 'v2.3.2' mod 'icingaweb2', :git => 'https://forge.softwareheritage.org/source/puppet-icinga-icingaweb2', :tag => 'v2.3.1' mod 'inifile', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-inifile', :ref => 'v4.0.0' mod 'java', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-java', :tag => 'v6.0.0' mod 'java_ks', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-java_ks', :tag => 'v3.1.0' mod 'kafka', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-kafka', :ref => 'v5.3.0' mod 'keycloak', :git => 'https://forge.softwareheritage.org/source/puppet-treydock-keycloak', :ref => 'v6.19.0' mod 'letsencrypt', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-letsencrypt', :ref => 'v5.0.0' mod 'locales', :git => 'https://forge.softwareheritage.org/source/puppet-saz-locales', :ref => 'v2.5.1' mod 'mysql', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-mysql', :ref => 'v10.3.0' mod 'nginx', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-nginx', :ref => 'v1.0.0' mod 'ntp', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-ntp', :ref => 'v8.2.0' mod 'php', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-php', :ref => 'v7.0.0' mod 'postgresql', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-postgresql', :ref => 'v6.4.0' mod 'pgbouncer', :git => 'https://forge.softwareheritage.org/source/puppet-covermymeds-pgbouncer', :ref => '9ec0d8a1255bbb309c2ff38f229167209cad496b' mod 'puppet', :git => 'https://forge.softwareheritage.org/source/puppet-theforeman-puppet', :ref => 'latest_passenger' mod 'puppetdb', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-puppetdb', :ref => '7.4.0' mod 'memcached', :git => 'https://forge.softwareheritage.org/source/puppet-saz-memcached', :ref => 'v3.4.0' mod 'rabbitmq', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-rabbitmq', :ref => 'v10.0.0' mod 'redis', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-redis', :ref => 'v6.1.0' mod 'resolv_conf', :git => 'https://forge.softwareheritage.org/source/puppet-saz-resolv_conf', :ref => 'v4.1.0' mod 'ssh', :git => 'https://forge.softwareheritage.org/source/puppet-saz-ssh', :ref => 'v6.0.0' mod 'stdlib', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-stdlib', :ref => 'v6.2.0' mod 'sudo', :git => 'https://forge.softwareheritage.org/source/puppet-saz-sudo', :ref => 'v6.0.0' mod 'systemd', :git => 'https://forge.softwareheritage.org/source/puppet-camptocamp-systemd', - :ref => '2.7.0' + :ref => '2.12.0' mod 'timezone', :git => 'https://forge.softwareheritage.org/source/puppet-saz-timezone', :ref => 'v5.1.1' mod 'unattended_upgrades', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-unattended_upgrades', :ref => 'v4.0.0' mod 'varnish', :git => 'https://forge.softwareheritage.org/source/puppet-claranet-varnish', :ref => 'bugfix/systemd-unit' mod 'vcsrepo', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-vcsrepo', :ref => 'v3.1.0' mod 'zookeeper', :git => 'https://forge.softwareheritage.org/source/puppet-deric-zookeeper', :ref => 'v0.8.7' diff --git a/site-modules/profile/manifests/swh/deploy/scheduler.pp b/site-modules/profile/manifests/swh/deploy/scheduler.pp index 62a5e7ae..6cc7fb40 100644 --- a/site-modules/profile/manifests/swh/deploy/scheduler.pp +++ b/site-modules/profile/manifests/swh/deploy/scheduler.pp @@ -1,125 +1,144 @@ # Deployment of swh-scheduler related utilities class profile::swh::deploy::scheduler { $config_dir = lookup('swh::deploy::scheduler::conf_dir') $config_file = lookup('swh::deploy::scheduler::conf_file') $user = lookup('swh::deploy::scheduler::user') $group = lookup('swh::deploy::scheduler::group') $config = lookup('swh::deploy::scheduler::config') $sentry_dsn = lookup('swh::deploy::scheduler::sentry_dsn', Optional[String], 'first', undef) $sentry_environment = lookup('swh::deploy::scheduler::sentry_environment', Optional[String], 'first', undef) $sentry_swh_package = lookup('swh::deploy::scheduler::sentry_swh_package', Optional[String], 'first', undef) $listener_log_level = lookup('swh::deploy::scheduler::listener::log_level') $task_broker = lookup('swh::deploy::scheduler::task_broker') $listener_service_name = 'swh-scheduler-listener' $listener_unit_name = "${listener_service_name}.service" $listener_unit_template = "profile/swh/deploy/scheduler/${listener_service_name}.service.erb" $runner_service_name = 'swh-scheduler-runner' $runner_priority_service_name = 'swh-scheduler-runner-priority' $packages = lookup('swh::deploy::scheduler::packages') $services = [ $listener_service_name, $runner_service_name, $runner_priority_service_name, ] include profile::swh::deploy::base_scheduler file {$config_file: ensure => present, owner => 'root', group => $group, mode => '0640', content => inline_template("<%= @config.to_yaml %>\n"), notify => Service[$services], require => File[$config_dir], } # Template uses variables # - $user # - $group # - $sentry_{dsn,environment,swh_package} # ::systemd::unit_file {$listener_unit_name: ensure => present, content => template($listener_unit_template), notify => Service[$listener_service_name], } service {$listener_service_name: ensure => running, enable => true, require => [ Package[$packages], File[$config_file], Systemd::Unit_File[$listener_unit_name], ], subscribe => Package[$packages], } ::profile::swh::deploy::scheduler::runner {$runner_service_name: user => $user, group => $group, packages => $packages, config_file => $config_file, sentry_dsn => $sentry_dsn, sentry_environment => $sentry_environment, sentry_swh_package => $sentry_swh_package, } ::profile::swh::deploy::scheduler::runner {$runner_priority_service_name: user => $user, group => $group, packages => $packages, config_file => $config_file, sentry_dsn => $sentry_dsn, sentry_environment => $sentry_environment, sentry_swh_package => $sentry_swh_package, priority => true, } # scheduler rpc server ::profile::swh::deploy::rpc_server {'scheduler': config_key => 'scheduler::remote', executable => 'swh.scheduler.api.server:make_app_from_configfile()', http_check_string => 'Software Heritage scheduler RPC server', } + # scheduler update metrics routine + + # Template uses variables + # - $user + # - $group + # - $config_file + # + $update_metrics_service_name = "swh-scheduler-update-metrics" + $update_metrics_unit_template = "profile/swh/deploy/scheduler/${update_metrics_service_name}.service.erb" + $update_metrics_timer_name = "${update_metrics_service_name}.timer" + $update_metrics_timer_template = "profile/swh/deploy/scheduler/${update_metrics_timer_name}.erb" + + ::systemd::timer { $update_metrics_timer_name: + timer_content => template($update_metrics_timer_template), + service_content => template($update_metrics_unit_template), + active => true, + enable => true, + require => Package[$packages], + } + # task archival cron $archive_config_file = lookup('swh::deploy::scheduler::archive::conf_file') $archive_config = lookup('swh::deploy::scheduler::archive::config') file {$archive_config_file: ensure => present, owner => 'root', group => $group, mode => '0640', content => inline_template("<%= @archive_config.to_yaml %>\n"), require => File[$config_dir], } cron {'archive_completed_oneshot_and_disabled_recurring_tasks': ensure => absent, user => $user, } profile::cron::d {'scheduler_archive_tasks': user => $user, command => "/usr/bin/swh scheduler --config-file ${archive_config_file} task archive", monthday => '1', hour => '0', minute => '0', require => [ Package[$packages], File[$archive_config_file], ], } - } diff --git a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-update-metrics.service.erb b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-update-metrics.service.erb new file mode 100644 index 00000000..8a4c26d4 --- /dev/null +++ b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-update-metrics.service.erb @@ -0,0 +1,17 @@ +# Scheduler Update Metrics unit file +# Managed by puppet class profile::swh::deploy::scheduler +# Changes will be overwritten + +[Unit] +Description=Software Heritage scheduler update-metrics +Requires=network.target +After=network.target + +[Service] +User=<%= @user %> +Group=<%= @group %> +Type=simple +ExecStart=/usr/bin/swh scheduler --config-file <%= @config_file %> origin update-metrics + +[Install] +WantedBy=multi-user.target diff --git a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-update-metrics.timer.erb b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-update-metrics.timer.erb new file mode 100644 index 00000000..5d16b9d5 --- /dev/null +++ b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-update-metrics.timer.erb @@ -0,0 +1,9 @@ +[Install] +WantedBy=timers.target + +[Timer] +OnCalendar=daily +Unit=swh-scheduler-update-metrics.service + +[Unit] +Description=regular scheduler update-metrics refresh