elasticsearch auto delete old indices

定在crontab 每天执行

crontab -e

* 2 * * * ~/autodelete.py

Python 代码如下

#!/usr/bin/env python
# encoding:utf-8

'''
'''

__author__ = 'moryaden at qq.com'

import sys
import argparse
import re
import os
import traceback
import logging.config
import datetime

# install with `pip install elasticsearch`
# for es 0.9x, install elasticsearch-py 0.4.x
import elasticsearch

def cfgLogging():
    logpath = os.path.join("log")

    if 'win' not in sys.platform:
        os.system("mkdir -p " + logpath)
    else:
        if not os.path.exists(logpath):
            os.makedirs(logpath)

    LOGGING = {
               'version': 1,
               'disable_existing_loggers': True,
               'formatters': {
                              'default': {'format': '[%(asctime)-25s] [%(relativeCreated)-15s] %(levelname)-10s pid:%(process)d %(message)s'},
                               # default': {
                               #           'format' : '%(asctime)s %(message)s',
                               #           'datefmt' : '%Y-%m-%d %H:%M:%S'
                               # }
                },
               'handlers': {
                            'console':{
                                       'level':'DEBUG',
                                       'class':'logging.StreamHandler',
                                       'formatter': 'default'
                            },
                            'file': {
                                     'level': 'DEBUG',
                                     'class': 'logging.handlers.RotatingFileHandler',
                                     'formatter': 'default',
                                     'filename' : os.path.join(logpath, 'runlog.log'),
                                     'maxBytes':    50 * 1024 * 1024,  # 10M
                                     'backupCount': 20,
                                     'encoding' : 'utf8',
                            }
                },
               'loggers' : {
                            '' : {
                                          'level' : 'DEBUG',
                                          'handlers' : ['console', 'file'],
                                          'propagate' : True
                            }
                }
    }
    logging.config.dictConfig(LOGGING)

def parseCmdOptions():
    parser = argparse.ArgumentParser()
    parser.add_argument('-n', "--nday", help = "delete indices beyond which day, could specify -1, -2 and -n",
        type = int, default = -7, choices = xrange(-10, 1))

    cmdoptions = parser.parse_args()

    return cmdoptions

def getDeadLineDate(day = -2):
    dt_today = datetime.datetime.today()
    time_delta = datetime.timedelta(day)
    dead_line_day = dt_today + time_delta
    return dead_line_day

def run():
    logger = logging.getLogger('main')
    cmdoptions = parseCmdOptions()

    es_hosts = ['10.0.0.1', ]
    es = elasticsearch.Elasticsearch(es_hosts)
    dead_line_day = getDeadLineDate(cmdoptions.nday)
    date_format = '%Y.%m.%d'

    d = es.indices.get_alias('logstash*')
    logger.info("I see all names {}".format(d.keys()))

    for name in d:
        m = re.search(r"^logstash.(.*)$", name)
        if m:
            s_date = m.group(1)
            logdate = datetime.datetime.strptime(s_date, date_format)
            if dead_line_day > logdate:
                ack = es.indices.delete(name)
                logger.info("deleting %s, ack={}".format(ack), name)
            else:
                logger.info("keeping %s", name)

def main():
    try:
        cfgLogging()
        logger = logging.getLogger("main")
        run()
    except SystemExit:
        pass
    except:
        logger.info('----->>>>>>>>>> {}'.format(traceback.format_exc()))

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        pass