Page MenuHomeSoftware Heritage
Paste P1004

elasticsearch index freeze/close
ActivePublic

Authored by olasd on Apr 12 2021, 10:31 AM.
import datetime
import iso8601
import elasticsearch
today = datetime.date.today()
days = lambda n: datetime.timedelta(days=n)
es = elasticsearch.Elasticsearch(hosts=['esnode1.internal.softwareheritage.org', 'esnode2.internal.softwareheritage.org', 'esnode3.internal.softwareheritage.org'], timeout=1200)
for l in sorted(es.cat.indices(h='i,sth,status').splitlines()):
i, throttled, status = l.split()
throttled = throttled == 'true'
if throttled and status != 'open':
continue
if i.startswith('.'):
continue
date = i.split('-')[-1]
if not date.startswith('20'):
continue
date = date.replace('.', '-')
date = iso8601.parse_date(date).date()
info = es.indices.get(i)[i]
shards = int(info['settings']['index']['number_of_shards'])
if not throttled and date < today - days(7):
print('freezing', i)
es.indices.freeze(i, wait_for_active_shards=shards)
status = 'open'
if status == 'open' and date < today - days(30):
print('closing', i)
es.indices.close(i)