diff --git a/manifests/swh/deploy/scheduler_updater.pp b/manifests/swh/deploy/scheduler_updater.pp new file mode 100644 index 0000000..474c647 --- /dev/null +++ b/manifests/swh/deploy/scheduler_updater.pp @@ -0,0 +1,32 @@ +# Deployment of swh-scheduler-updater related utilities + +class profile::swh::deploy::scheduler::updater { + # Package and backend configuration + $scheduler_updater_packages = ['python3-swh.scheduler.updater'] + + package {$scheduler_updater_packages: + ensure => latest, + } + + $backend_conf_dir = lookup('swh::deploy::scheduler::updater::backend::conf_dir') + $backend_conf_file = lookup('swh::deploy::scheduler::updater::backend::conf_file') + $backend_user = lookup('swh::deploy::scheduler::updater::backend::user') + $backend_group = lookup('swh::deploy::scheduler::updater::backend::group') + $backend_config = lookup('swh::deploy::scheduler::updater::backend::config') + + file {$backend_conf_dir: + ensure => directory, + owner => 'root', + group => $backend_group, + mode => '0755', + } + + file {$backend_config: + ensure => present, + owner => 'root', + group => $backend_group, + mode => '0640', + content => inline_template("<%= @backend_config.to_yaml %>\n"), + } + +} diff --git a/manifests/swh/deploy/scheduler_updater_consumer.pp b/manifests/swh/deploy/scheduler_updater_consumer.pp new file mode 100644 index 0000000..04e606c --- /dev/null +++ b/manifests/swh/deploy/scheduler_updater_consumer.pp @@ -0,0 +1,68 @@ +# Deployment of swh-scheduler-updater related utilities +class profile::swh::deploy::scheduler::updater::consumer { + include ::profile::swh::deploy::scheduler::updater + + # only ghtorrent so far + $consumer_conf_dir = lookup('swh::deploy::scheduler::updater::consumer::ghtorrent::conf_dir') + $consumer_conf_file = lookup('swh::deploy::scheduler::updater::consumer::ghtorrent::conf_file') + $consumer_user = lookup('swh::deploy::scheduler::updater::consumer::user') + $consumer_group = lookup('swh::deploy::scheduler::updater::consumer::group') + + file {$consumer_conf_dir: + ensure => directory, + owner => 'root', + group => $consumer_group, + mode => '0755', + } + + $consumer_config = lookup('swh::deploy::scheduler::updater::consumer::ghtorrent::config') + file {$consumer_conf_file: + ensure => present, + owner => 'root', + group => $consumer_group, + mode => '0640', + content => inline_template("<%= @consumer_config.to_yaml %>\n"), + } + + # service needed to forward port locally + + $local_port = lookup('swh::deploy::scheduler::updater::consumer::ghtorrent::port') + $ghtorrent_private_key_raw = lookup('swh::deploy::scheduler::updater::consumer::ghtorrent::private_key') + $ghtorrent_private_key = '~/.ssh/id-rsa-swh-ghtorrent' + + # write private key to access the ghtorrent infra + file {$ghtorrent_private_key: + ensure => present, + owner => $consumer_user, + group => $consumer_group, + mode => '0600', + content => inline_template("<%= @ghtorrent_private_key_raw %>"), + } + + $ghtorrent_service_name = 'ssh-ghtorrent' + $ghtorrent_unit_name = "${ghtorrent_service_name}.service" + # Service to open up the ghtorrent connection infra (no consumption) + ::systemd::unit_file {$ghtorrent_unit_name: + ensure => present, + content => template("profile/swh/deploy/scheduler/${ghtorrent_unit_name}.erb"), + } ~> service {$ghtorrent_service_name: + ensure => running, + enable => true, + require => File[$ghtorrent_private_key], + } + + # actual service consuming from ghtorrent + + ghtorrent_consumer_service = 'swh-scheduler-updater-consumer-ghtorrent.service' + ghtorrent_consumer_unit_name = "${ghtorrent_consumer_service_name}.service" + # Service to consume from ghtorrent + ::systemd::unit_file {$ghtorrent_consumer_unit_name: + ensure => present, + content => template("profile/swh/deploy/scheduler/${ghtorrent_consumer_unit_name}.erb"), + } ~> service {$ghtorrent_consumer_service: + ensure => stopped, + enable => true, + require => Service[$ghtorrent_service_name], + } + +} diff --git a/manifests/swh/deploy/scheduler_updater_writer.pp b/manifests/swh/deploy/scheduler_updater_writer.pp new file mode 100644 index 0000000..a1352cd --- /dev/null +++ b/manifests/swh/deploy/scheduler_updater_writer.pp @@ -0,0 +1,51 @@ +# Deployment of swh-scheduler-writer related utilities +class profile::swh::deploy::scheduler::updater::writer { + include ::profile::swh::deploy::scheduler::updater + + $writer_conf_dir = lookup('swh::deploy::scheduler::updater::writer::conf_dir') + $writer_conf_file = lookup('swh::deploy::scheduler::updater::writer::conf_file') + $writer_user = lookup('swh::deploy::scheduler::updater::writer::user') + $writer_group = lookup('swh::deploy::scheduler::updater::writer::group') + $writer_config = lookup('swh::deploy::scheduler::updater::writer::config') + + file {$writer_conf_dir: + ensure => directory, + owner => 'root', + group => $writer_group, + mode => '0755', + } + + file {$writer_config: + ensure => present, + owner => 'root', + group => $writer_group, + mode => '0640', + content => inline_template("<%= @writer_config.to_yaml %>\n"), + } + + # unit + timer + writer_service = 'swh-scheduler-updater-writer' + writer_unit_name = "${writer_service}.service" + # Service to consume from ghtorrent + ::systemd::unit_file {$writer_unit_name: + ensure => present, + content => template("profile/swh/deploy/scheduler/${writer_unit_name}.erb"), + } ~> service {$writer_service: + enable => true, + } + + writer_timer_period = lookup('swh::deploy::scheduler::writer::timer_period') + writer_timer = 'swh-scheduler-updater-writer' + writer_timer_unit_name = "${writer_timer}.timer" + ::systemd::unit_file {$writer_timer_unit_name: + ensure => present, + content => template("profile/swh/deploy/scheduler/${writer_timer_unit_name}.erb"), + } ~> service {$writer_service: + enable => true, + } ~> service {$writer_timer: + ensure => running, + enable => true, + require => Service[$ghtorrent_service_name], + } + +} diff --git a/templates/swh/deploy/scheduler/ssh-ghtorrent.service.erb b/templates/swh/deploy/scheduler/ssh-ghtorrent.service.erb new file mode 100644 index 0000000..b27ab94 --- /dev/null +++ b/templates/swh/deploy/scheduler/ssh-ghtorrent.service.erb @@ -0,0 +1,19 @@ +# SSH Ghtorrent unit file +# Managed by puppet class profile::swh::deploy::scheduler::updater::consumer +# Changes will be overwritten + +[Unit] +Description=SSH connection to GHTorrent infra +Requires=network.target +After=network.target + +[Service] +User=<%= @backend_user %> +Group=<%= @backend_group %> +Type=simple +ExecStart=ssh -i <%= @ghtorrent_private_key %> -L <%= @local_port %>:streamer.ghtorrent.org:5672 ghtorrent@streamer.ghtorrent.org +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/templates/swh/deploy/scheduler/swh-scheduler-updater-consumer-ghtorrent.service.erb b/templates/swh/deploy/scheduler/swh-scheduler-updater-consumer-ghtorrent.service.erb new file mode 100644 index 0000000..e38cf83 --- /dev/null +++ b/templates/swh/deploy/scheduler/swh-scheduler-updater-consumer-ghtorrent.service.erb @@ -0,0 +1,19 @@ +# Scheduler Updater Consumer unit file +# Managed by puppet class profile::swh::deploy::scheduler::updater::consumer +# Changes will be overwritten + +[Unit] +Description=Software Heritage Scheduler Updater Consumer +Requires=network.target ssh-ghtorrent.service +After=network.target ssh-ghtorrent.service + +[Service] +User=<%= @backend_user %> +Group=<%= @backend_group %> +Type=simple +ExecStart=/usr/bin/python3 -m swh.scheduler.updater.ghtorrent.cli +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/templates/swh/deploy/scheduler/swh-scheduler-updater-writer.service.erb b/templates/swh/deploy/scheduler/swh-scheduler-updater-writer.service.erb new file mode 100644 index 0000000..2e06381 --- /dev/null +++ b/templates/swh/deploy/scheduler/swh-scheduler-updater-writer.service.erb @@ -0,0 +1,17 @@ +# Scheduler Updater Writer unit file +# Managed by puppet class profile::swh::deploy::scheduler::updater::writer +# Changes will be overwritten + +[Unit] +Description=Software Heritage Scheduler Updater Writer +Requires=network.target +After=network.target + +[Service] +User=<%= @writer_user %> +Group=<%= @writer_group %> +Type=oneshot +ExecStart=echo /usr/bin/python3 -m swh.scheduler.updater.writer + +[Install] +WantedBy=multi-user.target diff --git a/templates/swh/deploy/scheduler/swh-scheduler-updater-writer.timer.erb b/templates/swh/deploy/scheduler/swh-scheduler-updater-writer.timer.erb new file mode 100644 index 0000000..8e44dd0 --- /dev/null +++ b/templates/swh/deploy/scheduler/swh-scheduler-updater-writer.timer.erb @@ -0,0 +1,18 @@ +# Scheduler Updater Writer Timer unit file +# Managed by puppet class profile::swh::deploy::scheduler::updater::writer +# Changes will be overwritten + +[Unit] +Description=Software Heritage Scheduler Updater Writer Timer + +[Timer] +# Time to wait after booting before we run first time +OnBootSec=10min +# periodic run +OnCalendar=<%= writer_timer_period %> +# of unit +Unit=<%= writer_unit_name %> + +[Install] +WantedBy=timers.target +