diff --git a/site-modules/profile/files/logstash/es_reopen_closed_indexes.sh b/site-modules/profile/files/logstash/es_reopen_closed_indexes.sh new file mode 100644 index 00000000..0ad4a179 --- /dev/null +++ b/site-modules/profile/files/logstash/es_reopen_closed_indexes.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# +# File managed by puppet (profile::logstash). All modifications will be lost. + +# Script to reopen and eventually unfreeze frozen indices in elasticsearch +# reason: +# - closed index or frozen index can't be written to +# - journalbeat replays old logs when a machine is rebooted which creates icinga alerts +# - source of this behavior ^ is not determined yet so we work around it with the following script + +ES_SERVER=192.168.100.61:9200 +LOGFILE=/var/log/logstash/logstash-plain.log +LIMIT=$1 + +function filter_index_name() { + # extract the index name for the lines: + # new log format : ... "type" =>" cluster_block_exception","reason " => " index [ swh_workers-7.15.2-2021.11.07 ]... + # old log format : ... ({type => cluster_block_exception, reason => index [ systemlogs-2021.11.09 ] blocked... + # .* reason "? => "? index \[ ([.a-z0-9_\-]+) \] .* + sed -r 's/.*reason"?=>"?index \[([.a-z0-9_\-]+)\].*/\1/g' | sort | uniq +} + +function log_indices() { + if [ -z "$1" ]; then + journalctl -x -u logstash | grep "cluster_block" | filter_index_name + else + tail -n$1 $LOGFILE | grep "cluster_block" | filter_index_name + fi +} + +while true; do + date + echo "Searching indices to reopen..." + INDICES="$(log_indices $LIMIT)" + echo "Found: ${INDICES}" + + for i in $INDICES; do + echo "Reopening $i" + printf "\tOpening : " + curl -f -s -XPOST "$ES_SERVER/${i}/_open" || echo -n "failure" + echo "" # new line after ES response + printf "\tUnfreeze: " + curl -f -s -XPOST "$ES_SERVER/${i}/_unfreeze" || echo -n "failure" + echo "" # new line after ES response + done + echo "Done" + sleep 30 +done diff --git a/site-modules/profile/manifests/logstash.pp b/site-modules/profile/manifests/logstash.pp index e9f25e62..8550df5f 100644 --- a/site-modules/profile/manifests/logstash.pp +++ b/site-modules/profile/manifests/logstash.pp @@ -1,55 +1,63 @@ # Install and configure logstash class profile::logstash { include ::java include ::profile::elastic::apt_config $version = sprintf('1:%s-1', lookup('elastic::elk_version')) $elasticsearch_hosts = lookup('logstash::elasticsearch::hosts') $listen_address = ip_for_network(lookup('kibana::listen_network')) package { 'logstash': ensure => $version, require => Class['java'], } apt::pin { 'logstash': packages => 'logstash', version => $version, priority => 1001, } file { '/etc/logstash/conf.d/input.conf': ensure => 'file', content => template('profile/logstash/input.conf.erb'), require => Package['logstash'], notify => Service['logstash'], } file { '/etc/logstash/conf.d/output.conf': ensure => 'file', content => template('profile/logstash/output.conf.erb'), require => Package['logstash'], notify => Service['logstash'], } file { '/etc/logstash/conf.d/filter.conf': ensure => 'file', content => template('profile/logstash/filter.conf.erb'), require => Package['logstash'], notify => Service['logstash'], } service { 'logstash': ensure => running, enable => true, require => [Package['logstash'], File['/etc/logstash/conf.d/input.conf'], File['/etc/logstash/conf.d/output.conf'], File['/etc/logstash/conf.d/filter.conf'] ], } + file { '/usr/local/bin/es_reopen_closed_indexes.sh': + ensure => 'file', + source => 'puppet:///modules/profile/logstash/es_reopen_closed_indexes.sh', + owner => 'root', + group => 'root', + mode => '0744' + } + include profile::icinga2::objects::logstash_checks }