diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -1353,6 +1353,7 @@ swh::remote_service::indexer::port: 5007 swh::remote_service::scheduler::port: 5008 swh::remote_service::search::port: 5010 +swh::remote_service::counters::port: 5011 # Default backend services. Override in specific sites if needed. Configurations # are split between read-only (the default) and writable storages. In most cases @@ -3197,6 +3198,49 @@ borg::encryption: repokey-blake2 +swh::deploy::base_counters::config_directory: "%{hiera('swh::conf_directory')}/counters" +swh::deploy::base_counters::user: swhstorage +swh::deploy::base_counters::group: swhstorage +swh::deploy::counters::conf_file: "%{hiera('swh::deploy::base_counters::config_directory')}/server.yml" +swh::deploy::counters::sentry_swh_package: swh.counters +swh::deploy::counters::sentry_environment: "%{alias('swh::deploy::environment')}" +swh::deploy::counters::sentry_dsn: "https://%{lookup('swh::deploy::counters::sentry_token')}@sentry.softwareheritage.org/19" +swh::deploy::counters::user: "%{alias('swh::deploy::base_counters::user')}" +swh::deploy::counters::group: "%{alias('swh::deploy::base_counters::group')}" +swh::deploy::counters::backend::listen::host: 127.0.0.1 +swh::deploy::counters::backend::listen::port: "%{alias('swh::remote_service::counters::port')}" +swh::deploy::counters::backend::workers: 2 +swh::deploy::counters::backend::reload_mercy: 3600 +swh::deploy::counters::backend::http_keepalive: 5 +swh::deploy::counters::backend::http_timeout: 3600 +swh::deploy::counters::backend::max_requests: 10000 +swh::deploy::counters::backend::max_requests_jitter: 1000 +swh::deploy::counters::backend::server_names: + - "%{::swh_hostname.internal_fqdn}" + - "%{::hostname}" + - 127.0.0.1 + - localhost + - "::1" + +swh::deploy::counters::config: + counters: + cls: redis + host: localhost:6379 + +swh::deploy::counters::journal_client::config_file: "%{lookup('swh::deploy::base_counters::config_directory')}/journal_client.yml" +swh::deploy::counters::journal_client::config: + counters: + cls: remote + hosts: http://localhost:5011 + journal: + brokers: "%{alias('swh::deploy::journal::brokers')}" + group_id: swh.counters.journal_client + prefix: swh.journal.objects + object_types: + - content + - origin + - revision + swh::deploy::base_search::config_directory: "%{hiera('swh::conf_directory')}/search" swh::deploy::base_search::user: swhstorage swh::deploy::base_search::group: swhstorage diff --git a/data/subnets/vagrant.yaml b/data/subnets/vagrant.yaml --- a/data/subnets/vagrant.yaml +++ b/data/subnets/vagrant.yaml @@ -155,6 +155,8 @@ host: search-esnode0.internal.staging.swh.network 10.168.130.90: host: search0.internal.staging.swh.network + 10.168.130.95: + host: counters0.internal.staging.swh.network 10.168.130.100: host: worker0.internal.staging.swh.network 10.168.130.101: diff --git a/manifests/site.pp b/manifests/site.pp --- a/manifests/site.pp +++ b/manifests/site.pp @@ -166,6 +166,10 @@ include role::swh_search_with_journal_client } +node /^counters\d\.internal\.staging\.swh\.network$/ { + include role::swh_counters_with_journal_client +} + node 'webapp.internal.staging.swh.network' { include role::swh_webapp } diff --git a/site-modules/profile/manifests/swh/deploy/base_counters.pp b/site-modules/profile/manifests/swh/deploy/base_counters.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/base_counters.pp @@ -0,0 +1,20 @@ +# Base class for counters related manifests +class profile::swh::deploy::base_counters { + $config_directory = lookup('swh::deploy::base_counters::config_directory') + $user = lookup('swh::deploy::base_counters::user') + $group = lookup('swh::deploy::base_counters::group') + + file {$config_directory: + ensure => 'directory', + owner => $user, + group => $group, + mode => '0755', + } + + $packages = ['python3-swh.counters'] + + package {$packages: + ensure => 'present', + } + +} diff --git a/site-modules/profile/manifests/swh/deploy/counters.pp b/site-modules/profile/manifests/swh/deploy/counters.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/counters.pp @@ -0,0 +1,13 @@ +# Deployment of the swh.counters.api server +class profile::swh::deploy::counters { + include ::profile::swh::deploy::base_counters + + class { '::redis': + bind => '127.0.0.1', + save_db_to_disk_interval => { '30' => '1' }, + } + + ::profile::swh::deploy::rpc_server {'counters': + executable => 'swh.counters.api.server:make_app_from_configfile()', + } +} diff --git a/site-modules/profile/manifests/swh/deploy/counters/journal_client.pp b/site-modules/profile/manifests/swh/deploy/counters/journal_client.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/counters/journal_client.pp @@ -0,0 +1,35 @@ +# Deployment of the swh.counters.journal_client +class profile::swh::deploy::counters::journal_client { + include ::profile::swh::deploy::base_counters + include ::profile::swh::deploy::journal + + $config_file = lookup('swh::deploy::counters::journal_client::config_file') + $config = lookup('swh::deploy::counters::journal_client::config') + + $user = lookup('swh::deploy::base_counters::user') + $group = lookup('swh::deploy::base_counters::group') + + $service_name = 'swh-counters-journal-client' + $unit_name = "${service_name}.service" + + file {$config_file: + ensure => present, + owner => 'root', + group => $group, + mode => '0644', + content => inline_template("<%= @config.to_yaml %>\n"), + notify => Service[$service_name], + } + + # Template uses variables + # - $user + # - $group + # + ::systemd::unit_file {$unit_name: + ensure => present, + content => template("profile/swh/deploy/journal/${unit_name}.erb"), + } ~> service {$service_name: + ensure => running, + enable => true, + } +} diff --git a/site-modules/profile/templates/swh/deploy/journal/swh-counters-journal-client.service.erb b/site-modules/profile/templates/swh/deploy/journal/swh-counters-journal-client.service.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/swh/deploy/journal/swh-counters-journal-client.service.erb @@ -0,0 +1,18 @@ +# Search Journal Client Template unit file +# Managed by puppet class profile::swh::deploy::counters::journal_client +# Changes will be overwritten + +[Unit] +Description=Software Heritage Counters Journal Client +After=network.target + +[Service] +User=<%= @user %> +Group=<%= @group %> +Type=simple +ExecStart=/usr/bin/swh counters --config-file <%= @config_file %> journal-client +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/site-modules/role/manifests/swh_counters_with_journal_client.pp b/site-modules/role/manifests/swh_counters_with_journal_client.pp new file mode 100644 --- /dev/null +++ b/site-modules/role/manifests/swh_counters_with_journal_client.pp @@ -0,0 +1,5 @@ +# sh counters and its journal client +class role::swh_counters_with_journal_client inherits role::swh_server { + include profile::swh::deploy::counters + include profile::swh::deploy::counters::journal_client +}