diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -3232,10 +3232,22 @@ prometheus::varnish::listen_network: "%{lookup('internal_network')}" prometheus::varnish::listen_port: 9104 +thanos::base::config_dir: "/etc/thanos" + thanos::release::version: 0.26.0 thanos::release::digest: cf5ea95e19388736df83f0959bd036b8ad400af233d03ae6f90decc05161dccc thanos::release::digest_type: sha256 +thanos::port::http: 19191 +thanos::port::grpc: 19090 + +thanos::sidecar::port_http: "%{lookup('thanos::port::http')}" +thanos::sidecar::port_grpc: "%{lookup('thanos::port::grpc')}" +thanos::query::port_http: "%{lookup('thanos::port::http')}" +thanos::query::config_filepath: "%{lookup('thanos::base::config_dir')}/query-sd.yaml" +thanos::gateway::port_http: 19192 +thanos::gateway::port_grpc: 19093 + thanos::tenant: "%{::subnet}" thanos::replica: "0" @@ -3249,6 +3261,12 @@ storage_account_key: "%{lookup('thanos::objstore::azure_account_key')}" container: "metrics-%{lookup('thanos::tenant')}-%{lookup('thanos::replica')}" +# Other puppet managed stores will be automatically dealt with +thanos::query::non_puppet_managed::stores: + # - pergamon.internal.softwareheritage.org:"%{lookup('thanos::sidecar::port_http')}" + # collected resource ^ + - mmca.softwareheritage.org:443 + grafana::db::database: grafana grafana::db::username: grafana # grafana::db::password in private-data diff --git a/data/subnets/vagrant.yaml b/data/subnets/vagrant.yaml --- a/data/subnets/vagrant.yaml +++ b/data/subnets/vagrant.yaml @@ -95,6 +95,8 @@ aliases: - inventory.internal.admin.swh.network - inventory.internal.softwareheritage.org + 10.168.50.90: + host: thanos.internal.admin.swh.network 10.168.100.18: host: banco.internal.softwareheritage.org aliases: diff --git a/manifests/site.pp b/manifests/site.pp --- a/manifests/site.pp +++ b/manifests/site.pp @@ -122,6 +122,10 @@ include role::swh_sentry } +node 'thanos.internal.admin.swh.network' { + include role::swh_thanos +} + node /^jenkins-debian\d+\.internal\.softwareheritage\.org$/ { include role::swh_ci_agent_debian } diff --git a/site-modules/profile/manifests/puppet/master.pp b/site-modules/profile/manifests/puppet/master.pp --- a/site-modules/profile/manifests/puppet/master.pp +++ b/site-modules/profile/manifests/puppet/master.pp @@ -103,5 +103,4 @@ minute => 'fqdn_rand', hour => 'fqdn_rand', } - } diff --git a/site-modules/profile/manifests/thanos/base.pp b/site-modules/profile/manifests/thanos/base.pp --- a/site-modules/profile/manifests/thanos/base.pp +++ b/site-modules/profile/manifests/thanos/base.pp @@ -15,6 +15,8 @@ $current_symlink = "${install_basepath}/current" + $config_dir = lookup('thanos::base::config_dir') + file { [$install_basepath, $install_dir]: ensure => 'directory', owner => $user, @@ -40,6 +42,12 @@ -> file {$current_symlink: ensure => 'link', target => $install_dir, - notify => Service[$service_name], + } + + file {$config_dir: + ensure => directory, + owner => $user, + group => 'prometheus', + mode => '0750', } } diff --git a/site-modules/profile/manifests/thanos/export_query_endpoint.pp b/site-modules/profile/manifests/thanos/export_query_endpoint.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/thanos/export_query_endpoint.pp @@ -0,0 +1,9 @@ +# Define exported resource for a thanos query/gateway endpoint +define profile::thanos::export_query_endpoint ( + String $grpc_address, +) { + + @@::profile::thanos::query_endpoint{"${facts['swh_hostname']['short']}_${name}": + grpc_address => $grpc_address + } +} diff --git a/site-modules/profile/manifests/thanos/prometheus_sidecar.pp b/site-modules/profile/manifests/thanos/prometheus_sidecar.pp --- a/site-modules/profile/manifests/thanos/prometheus_sidecar.pp +++ b/site-modules/profile/manifests/thanos/prometheus_sidecar.pp @@ -7,9 +7,15 @@ $objstore_config = lookup('thanos::objstore::config') - $config_dir = '/etc/thanos-sidecar' + $config_dir = '/etc/thanos' $objstore_config_file = "${config_dir}/objstore.yml" + $port_http = lookup('thanos::sidecar::port_http') + $port_grpc = lookup('thanos::sidecar::port_grpc') + + $internal_ip = ip_for_network(lookup('internal_network')) + $grpc_address = "${internal_ip}:${port_grpc}" + $sidecar_arguments = { tsdb => { path => '/var/lib/prometheus/metrics2', @@ -24,11 +30,10 @@ shipper => { 'upload-compacted' => true, }, - 'http-address' => '0.0.0.0:19191', - 'grpc-address' => '0.0.0.0:19090', + 'http-address' => "${internal_ip}:${port_http}", + 'grpc-address' => $grpc_address, } - file {$config_dir: ensure => directory, owner => 'root', @@ -45,6 +50,8 @@ content => inline_yaml($objstore_config), } + # Template uses: + # $sidecar_arguments systemd::unit_file {$unit_name: ensure => present, content => template('profile/thanos/thanos-sidecar.service.erb'), @@ -61,4 +68,8 @@ Class['profile::thanos::base'] ~> Service[$service_name] # Ensure prometheus is configured properly before starting the sidecar Exec['restart-prometheus'] -> Service[$service_name] + + ::profile::thanos::export_query_endpoint {"thanos-sidecar-${::fqdn}": + grpc_address => $grpc_address + } } diff --git a/site-modules/profile/manifests/thanos/query.pp b/site-modules/profile/manifests/thanos/query.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/thanos/query.pp @@ -0,0 +1,56 @@ +# Thanos query +class profile::thanos::query { + include profile::thanos::base + + $service_name = 'thanos-query' + $unit_name = "${service_name}.service" + + $port_http = lookup('thanos::query::port_http') + $non_puppet_managed_stores = lookup('thanos::query::non_puppet_managed::stores') + + $internal_ip = ip_for_network(lookup('internal_network')) + $config_filepath = lookup('thanos::query::config_filepath') + + concat::fragment { 'header': + target => $config_filepath, + content => "---\n- targets:\n", + order => 0, + tag => 'thanos', + require => File[$config_dir], + } + + $non_puppet_managed_stores.map | $store | { + concat::fragment { $store: + target => $config_filepath, + content => " - ${store}\n", + order => 1, + tag => 'thanos', + require => File[$config_dir], + } + } + + # Deal with collected resources + Concat <<| tag == 'thanos' |>> ~> Service[$service_name] + Concat::Fragment <<| tag == 'thanos' |>> ~> Service[$service_name] + + $query_arguments = { + "http-address" => "${internal_ip}:${port_http}", + "store.sd-files" => $config_filepath, + } + + systemd::unit_file {$unit_name: + ensure => present, + content => template("profile/thanos/${unit_name}.erb"), + require => Class['profile::thanos::base'], + notify => Service[$service_name], + } + + # Template uses: + # $query_arguments + service {$service_name: + ensure => 'running', + enable => true, + } + + Class['profile::thanos::base'] ~> Service[$service_name] +} diff --git a/site-modules/profile/manifests/thanos/query_endpoint.pp b/site-modules/profile/manifests/thanos/query_endpoint.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/thanos/query_endpoint.pp @@ -0,0 +1,27 @@ +# Define concrete resource for a thanos query/gateway endpoint +define profile::thanos::query_endpoint ( + String $grpc_address, +) { + + $config_filepath = lookup('thanos::query::config_filepath') + concat {$config_filepath: + ensure => present, + path => $config_filepath, + owner => $user, + group => 'prometheus', + mode => '0640', + ensure_newline => true, + order => 'numeric', + tag => 'thanos', + require => File[$::profile::thanos::base::config_dir] + } + + # sidecar grpc address pushed to query service configuration file + concat::fragment { $grpc_address: + target => $config_filepath, + content => " - ${grpc_address}\n", + order => 2, + tag => 'thanos', + } + +} diff --git a/site-modules/profile/templates/thanos/thanos-query.service.erb b/site-modules/profile/templates/thanos/thanos-query.service.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/thanos/thanos-query.service.erb @@ -0,0 +1,39 @@ +# File managed by puppet (class profile::thanos::query) +# Manual changes will be overwritten + +[Unit] +Description=Thanos query + +[Service] +Restart=on-failure +User=prometheus +ExecStart=/opt/thanos/current/thanos query <%= scope.call_function('flatten_to_argument_list', [@query_arguments]).join(" \\\n ") %> +ExecReload=/bin/kill -HUP $MAINPID +TimeoutStopSec=20s +SendSIGKILL=no + +# systemd hardening-options +AmbientCapabilities= +CapabilityBoundingSet= +DeviceAllow=/dev/null rw +DevicePolicy=strict +LimitMEMLOCK=0 +LimitNOFILE=8192 +LockPersonality=true +MemoryDenyWriteExecute=true +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +PrivateUsers=true +ProtectControlGroups=true +ProtectHome=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectSystem=full +RemoveIPC=true +RestrictNamespaces=true +RestrictRealtime=true +SystemCallArchitectures=native + +[Install] +WantedBy=multi-user.target diff --git a/site-modules/role/manifests/swh_thanos.pp b/site-modules/role/manifests/swh_thanos.pp new file mode 100644 --- /dev/null +++ b/site-modules/role/manifests/swh_thanos.pp @@ -0,0 +1,4 @@ +# Thanos role +class role::swh_thanos inherits role::swh_server { + include profile::thanos::query +}