diff --git a/site-modules/profile/manifests/thanos/base.pp b/site-modules/profile/manifests/thanos/base.pp index b6e2030b..fe825b28 100644 --- a/site-modules/profile/manifests/thanos/base.pp +++ b/site-modules/profile/manifests/thanos/base.pp @@ -1,55 +1,57 @@ # Base installation of thanos class profile::thanos::base { $user = 'root' $group = 'root' $version = lookup('thanos::release::version') $archive_url = "https://github.com/thanos-io/thanos/releases/download/v${version}/thanos-${version}.linux-amd64.tar.gz" $archive_digest = lookup('thanos::release::digest') $archive_digest_type = lookup('thanos::release::digest_type') $install_basepath = "/opt/thanos" $install_dir = "${install_basepath}/${version}" $archive_path = "${install_basepath}/${version}.tar.gz" $current_symlink = "${install_basepath}/current" $config_dir = lookup('thanos::base::config_dir') file { [$install_basepath, $install_dir]: ensure => 'directory', owner => $user, group => $group, mode => '0644', } archive { 'thanos': path => $archive_path, extract => true, extract_command => 'tar xzf %s --strip-components=1 --no-same-owner --no-same-permissions', source => $archive_url, extract_path => $install_dir, checksum => $archive_digest, checksum_type => $archive_digest_type, creates => "${install_dir}/thanos", cleanup => true, user => $user, group => $group, require => File[$install_dir], } -> file {$current_symlink: ensure => 'link', target => $install_dir, } + ~> Service <| tag == 'thanos' |> + file {$config_dir: ensure => directory, owner => $user, group => 'prometheus', mode => '0750', purge => true, recurse => true, } } diff --git a/site-modules/profile/manifests/thanos/compact.pp b/site-modules/profile/manifests/thanos/compact.pp index 23f4978b..02234777 100644 --- a/site-modules/profile/manifests/thanos/compact.pp +++ b/site-modules/profile/manifests/thanos/compact.pp @@ -1,48 +1,52 @@ # Thanos compact services (compaction and downscaling of historical metrics) class profile::thanos::compact { include profile::thanos::base include profile::thanos::objstore_configs $internal_ip = ip_for_network(lookup('internal_network')) $stores = lookup('thanos::stores') $config_dir = $::profile::thanos::base::config_dir $stores.each | $dataset_name, $service | { $port_http = $service['compact']['port-http'] $http_address = "${internal_ip}:${port_http}" $http_target = "${swh_hostname['internal_fqdn']}:${port_http}" $service_name = "thanos-compact@${dataset_name}" $unit_name = "${service_name}.service" ::systemd::dropin_file {"${service_name}/parameters.conf": ensure => present, unit => $unit_name, filename => 'parameters.conf', content => template('profile/thanos/compact-parameters.conf.erb'), notify => Service[$service_name], } service {$service_name: ensure => 'running', enable => true, - tag => ['thanos-compact', "thanos-objstore-${dataset_name}"], + tag => [ + 'thanos', + 'thanos-compact', + "thanos-objstore-${dataset_name}", + ], } ::profile::prometheus::export_scrape_config {"thanos-compact-${http_target}": target => $http_target, job => 'thanos_compact', labels => { dataset_name => $dataset_name, }, } } # Uses: $config_dir, $cert_paths systemd::unit_file {'thanos-compact@.service': ensure => present, content => template('profile/thanos/compact@.service.erb'), require => Class['profile::thanos::base'], } ~> Service <| tag == 'thanos-compact' |> } diff --git a/site-modules/profile/manifests/thanos/prometheus_sidecar.pp b/site-modules/profile/manifests/thanos/prometheus_sidecar.pp index fd1f006d..df4e6500 100644 --- a/site-modules/profile/manifests/thanos/prometheus_sidecar.pp +++ b/site-modules/profile/manifests/thanos/prometheus_sidecar.pp @@ -1,86 +1,86 @@ # Thanos prometheus sidecar class profile::thanos::prometheus_sidecar { include profile::thanos::base include profile::thanos::tls_certificate $service_name = 'thanos-sidecar' $unit_name = "${service_name}.service" $objstore_config = lookup('thanos::objstore::config') $objstore_config_file = "${::profile::thanos::base::config_dir}/objstore.yml" $port_http = lookup('thanos::sidecar::port_http') $port_grpc = lookup('thanos::sidecar::port_grpc') $internal_ip = ip_for_network(lookup('internal_network')) $grpc_address = "${internal_ip}:${port_grpc}" $grpc_target = "${swh_hostname['internal_fqdn']}:${port_grpc}" $cert_paths = $::profile::thanos::tls_certificate::cert_paths $sidecar_arguments = { tsdb => { path => '/var/lib/prometheus/metrics2', }, prometheus => { # use the listen address for the prometheus server url => "http://${::profile::prometheus::server::target}/", }, objstore => { 'config-file' => $objstore_config_file, }, shipper => { 'upload-compacted' => true, }, 'grpc-server-tls-cert' => $cert_paths['fullchain'], 'grpc-server-tls-key' => $cert_paths['privkey'], 'http-address' => "${internal_ip}:${port_http}", 'grpc-address' => $grpc_address, } file {$objstore_config_file: ensure => present, owner => 'root', group => 'prometheus', mode => '0640', content => inline_yaml($objstore_config), require => File[$::profile::thanos::base::config_dir], } # Template uses: # $sidecar_arguments systemd::unit_file {$unit_name: ensure => present, content => template('profile/thanos/thanos-sidecar.service.erb'), require => Class['profile::thanos::base'], notify => Service[$service_name] } service {$service_name: ensure => 'running', enable => true, require => [ Service['prometheus'], File[$cert_paths['fullchain']], File[$cert_paths['privkey']], ], + tag => 'thanos', } - Class['profile::thanos::base'] ~> Service[$service_name] # Ensure prometheus is configured properly before starting the sidecar Exec['restart-prometheus'] -> Service[$service_name] # Ensure service is restarted when the certs are renewed File[$cert_paths['fullchain']] ~> Service[$service_name] File[$cert_paths['privkey']] ~> Service[$service_name] ::profile::thanos::export_query_endpoint {"thanos-sidecar-${::fqdn}": grpc_address => $grpc_target } $http_target = "${swh_hostname['internal_fqdn']}:${port_http}" ::profile::prometheus::export_scrape_config {"thanos-sidecar-${::fqdn}": target => $http_target, job => 'thanos_sidecar', } } diff --git a/site-modules/profile/manifests/thanos/query.pp b/site-modules/profile/manifests/thanos/query.pp index 6b321b02..57ddb28e 100644 --- a/site-modules/profile/manifests/thanos/query.pp +++ b/site-modules/profile/manifests/thanos/query.pp @@ -1,75 +1,74 @@ # Thanos query class profile::thanos::query { include profile::thanos::base $service_name = 'thanos-query' $unit_name = "${service_name}.service" $port_http = lookup('thanos::query::port_http') $non_puppet_managed_stores = lookup('thanos::query::non_puppet_managed::stores') $internal_ip = ip_for_network(lookup('internal_network')) $config_filepath = lookup('thanos::query::config_filepath') concat {$config_filepath: ensure => present, path => $config_filepath, owner => $user, group => 'prometheus', mode => '0640', ensure_newline => true, order => 'numeric', tag => 'thanos', require => File[$::profile::thanos::base::config_dir], notify => Service[$service_name], } concat::fragment { 'header': target => $config_filepath, content => "---\n- targets:\n", order => 0, tag => 'thanos', require => File[$config_dir], } $non_puppet_managed_stores.map | $store | { concat::fragment { $store: target => $config_filepath, content => " - ${store}\n", order => 1, tag => 'thanos', require => File[$config_dir], } } # Deal with collected resources Profile::Thanos::Query_endpoint <<| |>> $query_arguments = { "http-address" => "${internal_ip}:${port_http}", "store.sd-files" => $config_filepath, "grpc-client-tls-secure" => true, "grpc-client-tls-ca" => '/etc/ssl/certs/ca-certificates.crt', } systemd::unit_file {$unit_name: ensure => present, content => template("profile/thanos/${unit_name}.erb"), require => Class['profile::thanos::base'], notify => Service[$service_name], } # Template uses: # $query_arguments service {$service_name: ensure => 'running', enable => true, + tag => 'thanos', } $http_target = "${swh_hostname['internal_fqdn']}:${port_http}" ::profile::prometheus::export_scrape_config {'thanos_query': target => $http_target, } - - Class['profile::thanos::base'] ~> Service[$service_name] } diff --git a/site-modules/profile/manifests/thanos/store.pp b/site-modules/profile/manifests/thanos/store.pp index 78c00112..26164e72 100644 --- a/site-modules/profile/manifests/thanos/store.pp +++ b/site-modules/profile/manifests/thanos/store.pp @@ -1,88 +1,92 @@ # Thanos store services (historical metrics access) class profile::thanos::store { include profile::thanos::base include profile::thanos::tls_certificate include profile::thanos::objstore_configs $cert_paths = $::profile::thanos::tls_certificate::cert_paths $internal_ip = ip_for_network(lookup('internal_network')) $stores = lookup('thanos::stores') $config_dir = $::profile::thanos::base::config_dir $stores.each | $dataset_name, $service | { $port_http = $service['store']['port-http'] $http_address = "${internal_ip}:${port_http}" $http_target = "${swh_hostname['internal_fqdn']}:${port_http}" $port_grpc = $service['store']['port-grpc'] $grpc_address = "${internal_ip}:${port_grpc}" $grpc_target = "${swh_hostname['internal_fqdn']}:${port_grpc}" $service_name = "thanos-store@${dataset_name}" $unit_name = "${service_name}.service" ::systemd::dropin_file {"${service_name}/parameters.conf": ensure => present, unit => $unit_name, filename => 'parameters.conf', content => template('profile/thanos/store-parameters.conf.erb'), notify => Service[$service_name], } # Cleanup old thanos-gateway service instances service {"thanos-gateway@${dataset_name}": ensure => stopped, enable => false, } -> service {$service_name: ensure => 'running', enable => true, require => [ File[$cert_paths['fullchain']], File[$cert_paths['privkey']], ], - tag => ['thanos-store', "thanos-objstore-${dataset_name}"], + tag => [ + 'thanos', + 'thanos-store', + "thanos-objstore-${dataset_name}", + ], } # And clean up drop-in files for old service instances -> file {"/etc/systemd/system/thanos-gateway@${dataset_name}.service.d": ensure => absent, recurse => true, force => true, } # Ensure service is restarted when the certs are renewed File[$cert_paths['fullchain']] ~> Service[$service_name] File[$cert_paths['privkey']] ~> Service[$service_name] # store service grpc address pushed to query service configuration file to access # historical data ::profile::thanos::export_query_endpoint {"thanos-store-${grpc_target}": grpc_address => $grpc_target } ::profile::prometheus::export_scrape_config {"thanos-store-${http_target}": target => $http_target, job => 'thanos_store', labels => { dataset_name => $dataset_name, }, } } # Uses: $config_dir, $cert_paths systemd::unit_file {'thanos-store@.service': ensure => present, content => template('profile/thanos/store@.service.erb'), require => Class['profile::thanos::base'], } ~> Service <| tag == 'thanos-store' |> # Cleanup old thanos-gateway service file Service <| tag == 'thanos-store' |> -> systemd::unit_file {'thanos-gateway@.service': ensure => absent, } }