diff --git a/data/defaults.yaml b/data/defaults.yaml --- a/data/defaults.yaml +++ b/data/defaults.yaml @@ -1787,50 +1787,6 @@ - "::1" swh::deploy::scheduler::remote::config: "%{alias('swh::deploy::scheduler::config::local')}" -swh::deploy::scheduler::updater::backend::conf_dir: "%{hiera('swh::conf_directory')}/backend" -swh::deploy::scheduler::updater::backend::conf_file: "%{hiera('swh::deploy::scheduler::updater::backend::conf_dir')}/scheduler-updater.yml" -swh::deploy::scheduler::updater::backend::user: swhscheduler -swh::deploy::scheduler::updater::backend::group: swhscheduler -swh::deploy::scheduler::updater::backend::db::host: db.internal.softwareheritage.org -swh::deploy::scheduler::updater::backend::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" -swh::deploy::scheduler::updater::backend::db::dbname: softwareheritage-scheduler-updater -swh::deploy::scheduler::updater::backend::db::user: swhscheduler -swh::deploy::scheduler::updater::backend::config: - scheduling_updater_db: "host=%{hiera('swh::deploy::scheduler::updater::backend::db::host')} port=%{hiera('swh::deploy::scheduler::updater::backend::db::port')} dbname=%{hiera('swh::deploy::scheduler::updater::backend::db::dbname')} user=%{hiera('swh::deploy::scheduler::updater::backend::db::user')} password=%{hiera('swh::deploy::scheduler::updater::backend::db::password')}" - cache_read_limit: 1000 - -swh::deploy::scheduler::updater::consumer::ghtorrent::conf_dir: "%{hiera('swh::conf_directory')}/backend" -swh::deploy::scheduler::updater::consumer::ghtorrent::conf_file: "%{hiera('swh::deploy::scheduler::updater::consumer::ghtorrent::conf_dir')}/ghtorrent.yml" -swh::deploy::scheduler::updater::consumer::user: swhscheduler -swh::deploy::scheduler::updater::consumer::group: swhscheduler - -swh::deploy::scheduler::updater::consumer::ghtorrent::user: streamer -swh::deploy::scheduler::updater::consumer::ghtorrent::port: 2765 - -swh::deploy::scheduler::updater::consumer::ghtorrent::config: - conn: - url: "amqp://%{hiera('swh::deploy::scheduler::updater::consumer::ghtorrent::user')}:%{hiera('swh::deploy::scheduler::updater::consumer::ghtorrent::password')}@localhost:%{hiera('swh::deploy::scheduler::updater::consumer::ghtorrent::port')}" - exchange_name: ght-streams - routing_key: evt.*.insert - queue_name: swh_queue - debug: no - batch_cache_write: 1000 - rabbitmq_prefetch_read: 1000 - -swh::deploy::scheduler::updater::writer::conf_dir: "%{hiera('swh::conf_directory')}/backend" -swh::deploy::scheduler::updater::writer::conf_file: "%{hiera('swh::deploy::scheduler::updater::writer::conf_dir')}/scheduler-updater-writer.yml" -swh::deploy::scheduler::updater::writer::user: swhscheduler -swh::deploy::scheduler::updater::writer::group: swhscheduler -# systemd-analyze calendar "*-*-* *:00:00" -swh::deploy::scheduler::updater::writer::timer_period: "00/4:00" -swh::deploy::scheduler::updater::writer::config: - scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" - scheduler_updater: - scheduling_updater_db: "host=%{hiera('swh::deploy::scheduler::updater::backend::db::host')} dbname=%{hiera('swh::deploy::scheduler::updater::backend::db::dbname')} user=%{hiera('swh::deploy::scheduler::updater::backend::db::user')} password=%{hiera('swh::deploy::scheduler::updater::backend::db::password')}" - cache_read_limit: 1000 - pause: 10 - verbose: no - swh::deploy::scheduler::archive::conf_dir: "%{hiera('swh::conf_directory')}/backend" swh::deploy::scheduler::archive::conf_file: "%{hiera('swh::deploy::scheduler::archive::conf_dir')}/elastic.yml" swh::deploy::scheduler::archive::user: "%{hiera('swh::deploy::scheduler::user')}" diff --git a/data/hostname/belvedere.internal.softwareheritage.org.yaml b/data/hostname/belvedere.internal.softwareheritage.org.yaml --- a/data/hostname/belvedere.internal.softwareheritage.org.yaml +++ b/data/hostname/belvedere.internal.softwareheritage.org.yaml @@ -20,11 +20,6 @@ host: "%{hiera('swh::deploy::scheduler::db::host')}" auth_user: "%{hiera('swh::deploy::db::pgbouncer::user::login')}" port: "%{hiera('swh::deploy::db::secondary::port')}" - # swh-scheduler-updater - - source_db: "%{hiera('swh::deploy::scheduler::updater::backend::db::dbname')}" - host: "%{hiera('swh::deploy::scheduler::updater::backend::db::host')}" - auth_user: "%{hiera('swh::deploy::db::pgbouncer::user::login')}" - port: "%{hiera('swh::deploy::db::secondary::port')}" # swh-vault - source_db: "%{hiera('swh::deploy::vault::db::dbname')}" host: "%{hiera('swh::deploy::vault::db::host')}" diff --git a/site-modules/profile/manifests/swh/deploy/scheduler_updater.pp b/site-modules/profile/manifests/swh/deploy/scheduler_updater.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/scheduler_updater.pp +++ /dev/null @@ -1,32 +0,0 @@ -# Deployment of swh-scheduler-updater related utilities - -class profile::swh::deploy::scheduler_updater { - # Package and backend configuration - $scheduler_updater_packages = ['python3-swh.scheduler.updater'] - - package {$scheduler_updater_packages: - ensure => present, - } - - $backend_conf_dir = lookup('swh::deploy::scheduler::updater::backend::conf_dir') - $backend_conf_file = lookup('swh::deploy::scheduler::updater::backend::conf_file') - $backend_user = lookup('swh::deploy::scheduler::updater::backend::user') - $backend_group = lookup('swh::deploy::scheduler::updater::backend::group') - $backend_config = lookup('swh::deploy::scheduler::updater::backend::config') - - # file {$backend_conf_dir: - # ensure => directory, - # owner => 'root', - # group => $backend_group, - # mode => '0755', - # } - - file {$backend_conf_file: - ensure => present, - owner => 'root', - group => $backend_group, - mode => '0640', - content => inline_template("<%= @backend_config.to_yaml %>\n"), - } - -} diff --git a/site-modules/profile/manifests/swh/deploy/scheduler_updater_consumer.pp b/site-modules/profile/manifests/swh/deploy/scheduler_updater_consumer.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/scheduler_updater_consumer.pp +++ /dev/null @@ -1,74 +0,0 @@ -# Deployment of swh-scheduler-updater related utilities -class profile::swh::deploy::scheduler_updater_consumer { - include ::profile::swh::deploy::scheduler_updater - - # only ghtorrent so far - $consumer_conf_dir = lookup('swh::deploy::scheduler::updater::consumer::ghtorrent::conf_dir') - $consumer_conf_file = lookup('swh::deploy::scheduler::updater::consumer::ghtorrent::conf_file') - $consumer_user = lookup('swh::deploy::scheduler::updater::consumer::user') - $consumer_group = lookup('swh::deploy::scheduler::updater::consumer::group') - - $packages = ['autossh'] - package {$packages: - ensure => present, - } - -# file {$consumer_conf_dir: -# ensure => directory, -# owner => 'root', -# group => $consumer_group, -# mode => '0755', -# } - - $consumer_config = lookup('swh::deploy::scheduler::updater::consumer::ghtorrent::config') - file {$consumer_conf_file: - ensure => present, - owner => 'root', - group => $consumer_group, - mode => '0640', - content => inline_template("<%= @consumer_config.to_yaml %>\n"), - } - - # service needed to forward port locally - - $local_port = lookup('swh::deploy::scheduler::updater::consumer::ghtorrent::port') - $ghtorrent_private_key_raw = lookup('swh::deploy::scheduler::updater::consumer::ghtorrent::private_key') - $ghtorrent_private_key = "/home/${consumer_user}/.ssh/id-rsa-swh-ghtorrent" - - # write private key to access the ghtorrent infra - file {$ghtorrent_private_key: - ensure => present, - owner => $consumer_user, - group => $consumer_group, - mode => '0600', - content => inline_template("<%= @ghtorrent_private_key_raw %>"), - } - - $ghtorrent_service_name = 'ssh-ghtorrent' - $ghtorrent_unit_name = "${ghtorrent_service_name}.service" - # Service to open up the ghtorrent connection infra (no consumption) - ::systemd::unit_file {$ghtorrent_unit_name: - ensure => present, - content => template("profile/swh/deploy/scheduler/${ghtorrent_unit_name}.erb"), - require => Package[$packages], - } ~> service {$ghtorrent_service_name: - ensure => stopped, - enable => true, - require => File[$ghtorrent_private_key], - } - - # actual service consuming from ghtorrent - - $ghtorrent_consumer_service = 'swh-scheduler-updater-consumer-ghtorrent' - $ghtorrent_consumer_unit_name = "${ghtorrent_consumer_service}.service" - # Service to consume from ghtorrent - ::systemd::unit_file {$ghtorrent_consumer_unit_name: - ensure => present, - content => template("profile/swh/deploy/scheduler/${ghtorrent_consumer_unit_name}.erb"), - } ~> service {$ghtorrent_consumer_service: - ensure => stopped, - enable => true, - require => Service[$ghtorrent_service_name], - } - -} diff --git a/site-modules/profile/manifests/swh/deploy/scheduler_updater_writer.pp b/site-modules/profile/manifests/swh/deploy/scheduler_updater_writer.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/scheduler_updater_writer.pp +++ /dev/null @@ -1,51 +0,0 @@ -# Deployment of swh-scheduler-writer related utilities -class profile::swh::deploy::scheduler_updater_writer { - include ::profile::swh::deploy::scheduler_updater - - $writer_conf_dir = lookup('swh::deploy::scheduler::updater::writer::conf_dir') - $writer_conf_file = lookup('swh::deploy::scheduler::updater::writer::conf_file') - $writer_user = lookup('swh::deploy::scheduler::updater::writer::user') - $writer_group = lookup('swh::deploy::scheduler::updater::writer::group') - $writer_config = lookup('swh::deploy::scheduler::updater::writer::config') - -# file {$writer_conf_dir: -# ensure => directory, -# owner => 'root', -# group => $writer_group, -# mode => '0755', -# } - - file {$writer_conf_file: - ensure => present, - owner => 'root', - group => $writer_group, - mode => '0640', - content => inline_template("<%= @writer_config.to_yaml %>\n"), - } - - # unit + timer - $writer_service = 'swh-scheduler-updater-writer' - $writer_unit_name = "${writer_service}.service" - # Service to consume from ghtorrent - ::systemd::unit_file {$writer_unit_name: - ensure => present, - content => template("profile/swh/deploy/scheduler/${writer_unit_name}.erb") - } ~> service {$writer_unit_name: - ensure => stopped, - enable => true, - require => File[$writer_conf_file], - } - - $writer_timer_period = lookup('swh::deploy::scheduler::updater::writer::timer_period') - $writer_timer = 'swh-scheduler-updater-writer' - $writer_timer_unit_name = "${writer_timer}.timer" - ::systemd::unit_file {$writer_timer_unit_name: - ensure => present, - content => template("profile/swh/deploy/scheduler/${writer_timer_unit_name}.erb"), - } ~> service {$writer_timer_unit_name: - ensure => stopped, - enable => true, - require => Service[$writer_unit_name], - } - -} diff --git a/site-modules/profile/templates/swh/deploy/scheduler/ssh-ghtorrent.service.erb b/site-modules/profile/templates/swh/deploy/scheduler/ssh-ghtorrent.service.erb deleted file mode 100644 --- a/site-modules/profile/templates/swh/deploy/scheduler/ssh-ghtorrent.service.erb +++ /dev/null @@ -1,20 +0,0 @@ -# SSH Ghtorrent unit file -# Managed by puppet class profile::swh::deploy::scheduler::updater::consumer -# Changes will be overwritten - -[Unit] -Description=SSH connection to GHTorrent infra -Requires=network-online.target -After=network-online.target - -[Service] -User=<%= @consumer_user %> -Group=<%= @consumer_group %> -Type=simple -Environment="AUTOSSH_GATETIME=0" -ExecStart=/usr/bin/autossh -M 0 -o "ServerAliveInterval 30" -o "ServerAliveCountMax 3" -i <%= @ghtorrent_private_key %> -NL <%= @local_port %>:streamer.ghtorrent.org:5672 ghtorrent@streamer.ghtorrent.org -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-updater-consumer-ghtorrent.service.erb b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-updater-consumer-ghtorrent.service.erb deleted file mode 100644 --- a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-updater-consumer-ghtorrent.service.erb +++ /dev/null @@ -1,19 +0,0 @@ -# Scheduler Updater Consumer unit file -# Managed by puppet class profile::swh::deploy::scheduler::updater::consumer -# Changes will be overwritten - -[Unit] -Description=Software Heritage Scheduler Updater Consumer -Requires=network.target ssh-ghtorrent.service -After=network.target ssh-ghtorrent.service - -[Service] -User=<%= @consumer_user %> -Group=<%= @consumer_group %> -Type=simple -ExecStart=/usr/bin/python3 -m swh.scheduler.updater.ghtorrent.cli -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-updater-writer.service.erb b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-updater-writer.service.erb deleted file mode 100644 --- a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-updater-writer.service.erb +++ /dev/null @@ -1,17 +0,0 @@ -# Scheduler Updater Writer unit file -# Managed by puppet class profile::swh::deploy::scheduler::updater::writer -# Changes will be overwritten - -[Unit] -Description=Software Heritage Scheduler Updater Writer -Requires=network.target -After=network.target - -[Service] -User=<%= @writer_user %> -Group=<%= @writer_group %> -Type=oneshot -ExecStart=/usr/bin/python3 -m swh.scheduler.updater.writer - -[Install] -WantedBy=multi-user.target diff --git a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-updater-writer.timer.erb b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-updater-writer.timer.erb deleted file mode 100644 --- a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-updater-writer.timer.erb +++ /dev/null @@ -1,18 +0,0 @@ -# Scheduler Updater Writer Timer unit file -# Managed by puppet class profile::swh::deploy::scheduler::updater::writer -# Changes will be overwritten - -[Unit] -Description=Software Heritage Scheduler Updater Writer Timer - -[Timer] -# Time to wait after booting before we run first time -OnBootSec=10min -# periodic run -OnCalendar=<%= @writer_timer_period %> -# of unit -Unit=<%= @writer_unit_name %> - -[Install] -WantedBy=timers.target - diff --git a/site-modules/role/manifests/swh_scheduler.pp b/site-modules/role/manifests/swh_scheduler.pp --- a/site-modules/role/manifests/swh_scheduler.pp +++ b/site-modules/role/manifests/swh_scheduler.pp @@ -4,6 +4,4 @@ # Scheduler include profile::rabbitmq include profile::swh::deploy::scheduler - include profile::swh::deploy::scheduler_updater_consumer - include profile::swh::deploy::scheduler_updater_writer }