Page MenuHomeSoftware Heritage
Paste P1135

es_open_unfreeze_from_journalctl.sh and es_open_unfreeze.sh:
ActivePublic
Actions

Authored by ardumont on Aug 27 2021, 2:37 PM.
Manual installation so far.
es_open_unfreeze_from_journalctl.sh:
```
root@logstash0:~# cat /usr/local/bin/es_open_unfreeze_from_journalctl.sh
#!/bin/bash
set -x
# Script to reopen and eventually unfreeze frozen indices in elasticsearch
# reason:
# - closed index or frozen index can't be written to
# - journalbeat replays old logs when a machine is rebooted which creates icinga alerts
# - source of this behavior ^ is not determined yet so we work around it with the following script
ES_SERVER=192.168.100.61:9200
LIMIT=$1
function filter_index_name() {
awk '{print $17}' | sort | uniq | tr -d '[' | tr -d ']'
}
function log_indices() {
if [ -z "$1" ]; then
journalctl -x -u logstash | grep "cluster_block" | filter_index_name
else
tail -n$1 $LOGFILE | grep "cluster_block" | filter_index_name
fi
}
while true; do
date
log_indices $LIMIT | xargs -r -t -n1 -i{} curl -s -XPOST http://$ES_SERVER/{}/_open;
log_indices $LIMIT | xargs -r -t -n1 -i{} curl -s -XPOST $ES_SERVER/{}/_unfreeze;
sleep 30
done
```
es_open_unfreeze.sh:
```
root@logstash0:~# cat /usr/local/bin/es_open_unfreeze.sh
#!/bin/bash
set -x
# Script to reopen and eventually unfreeze frozen indices in elasticsearch
# reason:
# - closed index or frozen index can't be written to
# - journalbeat replays old logs when a machine is rebooted which creates icinga alerts
# - source of this behavior ^ is not determined yet so we work around it with the following script
ES_SERVER=192.168.100.61:9200
LOGFILE=/var/log/logstash/logstash-plain.log
LIMIT=$1
function filter_index_name() {
awk '{print $12}' | sort | uniq | tr -d '[' | tr -d ']'
}
function log_indices() {
if [ -z "$1" ]; then
grep "cluster_block" $LOGFILE | filter_index_name
else
tail -n$1 $LOGFILE | grep "cluster_block" | filter_index_name
fi
}
while true; do
date
log_indices $LIMIT | xargs -r -t -n1 -i{} curl -s -XPOST http://$ES_SERVER/{}/_open;
log_indices $LIMIT | xargs -r -t -n1 -i{} curl -s -XPOST $ES_SERVER/{}/_unfreeze;
sleep 30
done
```
j
About · Developer information · Legal