Manual installation so far. es_open_unfreeze_from_journalctl.sh: ``` root@logstash0:~# cat /usr/local/bin/es_open_unfreeze_from_journalctl.sh #!/bin/bash set -x # Script to reopen and eventually unfreeze frozen indices in elasticsearch # reason: # - closed index or frozen index can't be written to # - journalbeat replays old logs when a machine is rebooted which creates icinga alerts # - source of this behavior ^ is not determined yet so we work around it with the following script ES_SERVER=192.168.100.61:9200 LIMIT=$1 function filter_index_name() { awk '{print $17}' | sort | uniq | tr -d '[' | tr -d ']' } function log_indices() { if [ -z "$1" ]; then journalctl -x -u logstash | grep "cluster_block" | filter_index_name else tail -n$1 $LOGFILE | grep "cluster_block" | filter_index_name fi } while true; do date log_indices $LIMIT | xargs -r -t -n1 -i{} curl -s -XPOST http://$ES_SERVER/{}/_open; log_indices $LIMIT | xargs -r -t -n1 -i{} curl -s -XPOST $ES_SERVER/{}/_unfreeze; sleep 30 done ``` es_open_unfreeze.sh: ``` root@logstash0:~# cat /usr/local/bin/es_open_unfreeze.sh #!/bin/bash set -x # Script to reopen and eventually unfreeze frozen indices in elasticsearch # reason: # - closed index or frozen index can't be written to # - journalbeat replays old logs when a machine is rebooted which creates icinga alerts # - source of this behavior ^ is not determined yet so we work around it with the following script ES_SERVER=192.168.100.61:9200 LOGFILE=/var/log/logstash/logstash-plain.log LIMIT=$1 function filter_index_name() { awk '{print $12}' | sort | uniq | tr -d '[' | tr -d ']' } function log_indices() { if [ -z "$1" ]; then grep "cluster_block" $LOGFILE | filter_index_name else tail -n$1 $LOGFILE | grep "cluster_block" | filter_index_name fi } while true; do date log_indices $LIMIT | xargs -r -t -n1 -i{} curl -s -XPOST http://$ES_SERVER/{}/_open; log_indices $LIMIT | xargs -r -t -n1 -i{} curl -s -XPOST $ES_SERVER/{}/_unfreeze; sleep 30 done ``` j