curl -s http://$ES_NODE/_cat/indices\?s=index | grep -v close | grep systemlogs | awk '{print $3}' | grep 2020.05 | xargs -n1 -t -i{} curl -XPOST http://${ES_NODE}/{}/_close 
curl -s http://$ES_NODE/_cat/indices\?s=index | grep -v close | grep systemlogs | awk '{print $3}' | grep 2020.06 | xargs -n1 -t -i{} curl -XPOST http://${ES_NODE}/{}/_close

Apr 08 10:30:24 logstash0 logstash[1605158]: [2021-04-08T10:30:24,052][WARN ][logstash.outputs.elasticsearch][main][62d11c4234b8981da77a97955da92ac9de92b9a6dcd4582f407face31fd5c664] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"systemlogs-2021.04.08", :routing=>nil, :_type=>"_doc"}, #<LogStash::Event:0x2ec8df34>], :response=>{"index"=>{"_index"=>"systemlogs-2021.04.08", "_type"=>"_doc", "_id"=>nil, "status"=>400, "error"=>{"type"=>"validation_exception", "reason"=>"Validation Failed: 1: this action would add [2] total shards, but this cluster currently has [3000]/[3000] maximum shards open;"}}}}
Apr 08 10:30:24 logstash0 logstash[1605158]: [2021-04-08T10:30:24,052][WARN ][logstash.outputs.elasticsearch][main][62d11c4234b8981da77a97955da92ac9de92b9a6dcd4582f407face31fd5c664] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"systemlogs-2021.04.08", :routing=>nil, :_type=>"_doc"}, #<LogStash::Event:0x26cfcf58>], :response=>{"index"=>{"_index"=>"systemlogs-2021.04.08", "_type"=>"_doc", "_id"=>nil, "status"=>400, "error"=>{"type"=>"validation_exception", "reason"=>"Validation Failed: 1: this action would add [2] total shards, but this cluster currently has [3000]/[3000] maximum shards open;"}}}}
Apr 08 10:30:24 logstash0 logstash[1605158]: [2021-04-08T10:30:24,053][WARN ][logstash.outputs.elasticsearch][main][62d11c4234b8981da77a97955da92ac9de92b9a6dcd4582f407face31fd5c664] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"systemlogs-2021.04.08", :routing=>nil, :_type=>"_doc"}, #<LogStash::Event:0x1ddd876d>], :response=>{"index"=>{"_index"=>"systemlogs-2021.04.08", "_type"=>"_doc", "_id"=>nil, "status"=>400, "error"=>{"type"=>"validation_exception", "reason"=>"Validation Failed: 1: this action would add [2] total shards, but this cluster currently has [3000]/[3000] maximum shards open;"}}}}

❯ ./random_search.sh                                                                                        12:36:37

cat >/tmp/config.json <<EOF
{
  "index" : {
"translog.sync_interval" : null,
"translog.durability": null,
"refresh_interval": null
  }
}
EOF

❯ curl -s http://192.168.100.81:9200/origin/_settings\?pretty
{
  "origin" : {
    "settings" : {
      "index" : {
        "refresh_interval" : "60s",
        "number_of_shards" : "90",
        "translog" : {
          "sync_interval" : "60s",
          "durability" : "async"
        },
        "provided_name" : "origin",
        "creation_date" : "1608761881782",
        "number_of_replicas" : "1",
        "uuid" : "Mq8dnlpuRXO4yYoC6CTuQw",
        "version" : {
          "created" : "7090399"
        }
      }
    }
  }
}
❯ curl -s -H "Content-Type: application/json" -XPUT http://192.168.100.81:9200/origin/_settings\?pretty -d @/tmp/config.json
{
  "acknowledged" : true
}
❯ curl -s http://192.168.100.81:9200/origin/_settings\?pretty
{
  "origin" : {
    "settings" : {
      "index" : {
        "creation_date" : "1608761881782",
        "number_of_shards" : "90",
        "number_of_replicas" : "1",
        "uuid" : "Mq8dnlpuRXO4yYoC6CTuQw",
        "version" : {
          "created" : "7090399"
        },
        "provided_name" : "origin"
      }
    }
  }
}

❯ ./random_search.sh

export ES_SERVER=192.168.100.81:9200
curl -XPUT -H "Content-Type: application/json" http://$ES_SERVER/_index_template/origin\?pretty -d '{"index_patterns": "origin", "template": {"settings": { "index": { "number_of_replicas":1, "number_of_shards": 90 } } } } '

apt update && apt install linux-image-amd64 linux-headers-amd64 
# reboot to upgrade the kernel
apt install libnvpair1linux libuutil1linux libzfs2linux libzpool2linux zfs-dkms zfsutils-linux zfs-zed
systemctl stop elasticsearch
rm -rf /srv/elasticsearch/nodes/0
zpool create -O atime=off -m /srv/elasticsearch/nodes elasticsearch-data /dev/vdb
chown elasticsearch: /srv/elasticsearch/nodes

root@search-esnode0:~# curl -XPOST -H "Content-Type: application/json" http://${ES_SERVER}/_reindex\?pretty\&refresh=true\&requests_per_second=-1\&\&wait_for_completion=true -d @/tmp/reindex-production.json    
{
  "took" : 10215280,
  "timed_out" : false,
  "total" : 91517657,
  "updated" : 0,
  "created" : 91517657,
  "deleted" : 0,
  "batches" : 91518,
  "version_conflicts" : 0,
  "noops" : 0,
  "retries" : {
    "bulk" : 0,
    "search" : 0
  },
  "throttled_millis" : 0,
  "requests_per_second" : -1.0,
  "throttled_until_millis" : 0,
  "failures" : [ ]
}

cat >/tmp/config.json <<EOF
{
  "index" : {
    "translog.sync_interval" : "60s",
	"translog.durability": "async",
	"refresh_interval": "60s"
  }
}
EOF