Changeset View
Changeset View
Standalone View
Standalone View
site-modules/profile/files/elasticsearch/elasticsearch_close_index.py
| #!/usr/bin/env python3 | #!/usr/bin/env python3 | ||||
| import click | |||||
| import datetime | import datetime | ||||
| import click | |||||
| import iso8601 | import iso8601 | ||||
| import elasticsearch | import elasticsearch | ||||
| @click.command() | @click.command() | ||||
| @click.option('--host', '-h', 'hosts', | @click.option('--host', '-h', 'hosts', | ||||
| multiple=True, | multiple=True, | ||||
| help="Elasticsearch node instances") | help="Elasticsearch node instances") | ||||
| @click.option('--timeout', '-t', default=1200) | @click.option('--timeout', '-t', default=1200) | ||||
| @click.option('--freeze-after-days', '-f', default=7) | |||||
| @click.option('--close-after-days', '-c', default=30) | @click.option('--close-after-days', '-c', default=30) | ||||
| def main(hosts, timeout, freeze_after_days, close_after_days): | def main(hosts, timeout, close_after_days): | ||||
| """Janitor script to manage (freeze or close) indices when respective date threshold are | """Janitor script to close indices when date thresholds are | ||||
| exceeded. | exceeded. | ||||
| """ | """ | ||||
| today = datetime.date.today() | today = datetime.date.today() | ||||
| days = lambda n: datetime.timedelta(days=n) | close_after_days = datetime.timedelta(days=close_after_days) | ||||
| if not hosts: | if not hosts: | ||||
| raise ValueError("Provide a list of elasticsearch nodes") | raise ValueError("Provide a list of elasticsearch nodes") | ||||
| es = elasticsearch.Elasticsearch(hosts=hosts, timeout=timeout) | es = elasticsearch.Elasticsearch(hosts=hosts, timeout=timeout) | ||||
| for l in sorted(es.cat.indices(h='i,sth,status').splitlines()): | for line in sorted(es.cat.indices(h='i,status').splitlines()): | ||||
| i, throttled, status = l.split() | i, status = line.split() | ||||
| throttled = throttled == 'true' | |||||
| if throttled and status != 'open': | |||||
| continue | |||||
| # ignore dot-prefixed indexes (e.g. kibana settings) | # ignore dot-prefixed indexes (e.g. kibana settings) | ||||
| if i.startswith('.'): | if i.startswith('.'): | ||||
| continue | continue | ||||
| date = i.split('-')[-1] | date = i.split('-')[-1] | ||||
| if not date.startswith('20'): | if not date.startswith('20'): | ||||
| continue | continue | ||||
| date = date.replace('.', '-') | date = date.replace('.', '-') | ||||
| date = iso8601.parse_date(date).date() | date = iso8601.parse_date(date).date() | ||||
| info = es.indices.get(i)[i] | |||||
| shards = int(info['settings']['index']['number_of_shards']) | |||||
| if not throttled and date < today - days(freeze_after_days): | |||||
| print('freezing', i) | |||||
| es.indices.freeze(i, wait_for_active_shards=shards) | |||||
| status = 'open' | |||||
| if status == 'open' and date < today - days(close_after_days): | if status == 'open' and date < today - close_after_days: | ||||
| print('closing', i) | print('closing', i) | ||||
| es.indices.close(i) | es.indices.close(i) | ||||
| if __name__ == '__main__': | if __name__ == '__main__': | ||||
| main() | main() | ||||