Centos6.7 ELK日志系统部署

Centos6.7 ELK日志系统部署

原文地址：http://www.cnblogs.com/caoguo/p/4991602.html

一. 环境

elk服务器：192.168.55.134

logstash日志采集端：192.168.55.132

二.安装JDK

[root@elk01 ~]# cd /usr/local/src
[root@elk01 src]# wget http://download.oracle.com/otn-pub/java/jdk/8u65-b17/jdk-8u65-linux-x64.tar.gz?AuthParam=1447919869_29a658de74feaeda612894dc77923aa4
[root@elk01 src]# tar zxvf jdk-8u65-linux-x64.tar.gz
[root@elk01 src]# mv jdk1..0_65/ /usr/local/
[root@elk01 ~]# vi /etc/profile
JAVA_HOME=/usr/local/jdk1..0_20
export JAVA_HOME
PATH=$JAVA_HOME/bin:$PATH
export PATH

[root@elk01 ~]# source /etc/profile

#检查下

三.redis安装

#### Redis install ####

[root@elk01 src]#  yum install -y wget gcc make tcl
[root@elk01 src]# http://download.redis.io/releases/redis-3.0.5.tar.gz
[root@elk01 src]# cd redis-3.0.
[root@elk01 redis-3.0.]# make
[root@elk01 redis-3.0.]# make install
[root@elk01 redis-3.0.]# cp redis.conf /etc/
[root@elk01 redis-3.0.]# touch /etc/init.d/redis
[root@elk01 redis-3.0.]# chmod  /etc/init.d/redis

[root@elk01 redis-3.0.]# vi /etc/init.d/redis
#!/bin/bash
#
# Init file for redis
#
# chkconfig: -
# description: redis daemon
#
# processname: redis
# config: /etc/redis.conf
# pidfile: /var/run/redis.pid
source /etc/init.d/functions
#BIN="/usr/local/bin"
BIN="/usr/local/bin"
CONFIG="/etc/redis.conf"
PIDFILE="/var/run/redis.pid"
### Read configuration
[ -r "$SYSCONFIG" ] && source "$SYSCONFIG"
RETVAL=
prog="redis-server"
desc="Redis Server"
start() {
        if [ -e $PIDFILE ];then
             echo "$desc already running...."
             exit
        fi
        echo -n $"Starting $desc: "
        daemon $BIN/$prog $CONFIG
        RETVAL=$?
        echo
        [ $RETVAL -eq  ] && touch /var/lock/subsys/$prog
        return $RETVAL
}
stop() {
        echo -n $"Stop $desc: "
        killproc $prog
        RETVAL=$?
        echo
        [ $RETVAL -eq  ] && rm -f /var/lock/subsys/$prog $PIDFILE
        return $RETVAL
}
restart() {
        stop
        start
}
case "$1" in
  start)
        start
        ;;
  stop)
        stop
        ;;
  restart)
        restart
        ;;
  condrestart)
        [ -e /var/lock/subsys/$prog ] && restart
        RETVAL=$?
        ;;
  status)
        status $prog
        RETVAL=$?
        ;;
   *)
        echo $"Usage: $0 {start|stop|restart|condrestart|status}"
        RETVAL=
esac
exit $RETVAL

[root@elk01 redis-3.0.]# chkconfig redis on
[root@elk01 redis-3.0.]# /etc/init.d/redis start

#redis连接测试

四.logstash indexer服务安装配置

#### Logstash 从redis取数据到elasticsearch ####
[root@elk01 src]# wget https://download.elastic.co/logstash/logstash/logstash-2.0.0.tar.gz
[root@elk01 src]# tar zxf logstash-2.0..tar.gz -C /usr/local/
[root@elk01 ~]# /usr/local/logstash-2.0./bin/logstash -e 'input { stdin { } } output { stdout {} }'

[root@elk01 logstash-2.0.]# vi logstash_indexer.conf
input {
  redis {
      host => 'localhost'
      data_type => 'list'
      port => ''
      key => 'logstash:redis'
      type => 'redis-input'
  }
}

output {
  elasticsearch {
      hosts => 'localhost'
  }
}

[root@elk01 logstash-2.0.]# bin/logstash -f logstash_indexer.conf
[root@elk01 elasticsearch]# redis-cli monitor
OK
1448364122.959182 [ 127.0.0.1:] "rpush" "logstash:redis" "{\"message\":\"Nov 24 19:22:02 elk01 yum[3074]: Erased: httpd\",\"@version\":\"1\",\"@timestamp\":\"2015-11-24T11:22:02.553Z\",\"host\":\"0.0.0.0\",\"path\":\"/var/log/messages\",\"type\":\"messages\"}"

# redis-cli monitor 看到的输出

五. elasticsearch 安装配置

#### Elasticsearch ####
[root@elk01 src]# tar zxf elasticsearch-2.0..tar.gz
[root@elk01 src]# mv elasticsearch-2.0. /usr/local/elasticsearch
[root@elk01 src]# useradd elasticsearch -d /usr/local/elasticsearch -s /sbin/nologin

[root@elk01 ~]# vi /etc/sysconfig/elasticsearch
# Directory where the Elasticsearch binary distribution resides
ES_HOME=/usr/local/elasticsearch

# Heap Size (defaults to 256m min, 1g max)
#ES_HEAP_SIZE=2g

# Heap new generation
#ES_HEAP_NEWSIZE=

# max direct memory
#ES_DIRECT_SIZE=

# Additional Java OPTS
#ES_JAVA_OPTS=

# Maximum number of open files
MAX_OPEN_FILES=

# Maximum amount of locked memory
#MAX_LOCKED_MEMORY=

# Maximum number of VMA (Virtual Memory Areas) a process can own
MAX_MAP_COUNT=

# Elasticsearch log directory
LOG_DIR=/var/log/elasticsearch

# Elasticsearch data directory
DATA_DIR=/usr/local/elasticsearch/data

# Elasticsearch work directory
WORK_DIR=/tmp/elasticsearch

# Elasticsearch conf directory
CONF_DIR=/etc/elasticsearch

# Elasticsearch configuration file (elasticsearch.yml)
CONF_FILE=/etc/elasticsearch/elasticsearch.yml

# User to run as, change this to a specific elasticsearch user if possible
# Also make sure, this user can write into the log directories in case you change them
# This setting only works for the init script, but has to be configured separately for systemd startup
ES_USER=elasticsearch

# Configure restart on package upgrade (true, every other setting will lead to not restarting)
#RESTART_ON_UPGRADE=true

[root@elk01 ~]# mkdir -p /var/run/elasticsearch
[root@elk01 ~]# mkdir -p /var/log/elasticsearch
[root@elk01 ~]# mkdir -p /usr/local/elasticsearch/data
[root@elk01 ~]# mkdir -p /usr/local/elasticsearch/plugins
[root@elk01 ~]# mkdir -p /usr/local/elasticsearch/config/scripts

[root@elk01 ~]# chown -R elasticsearch.elasticsearch /var/run/elasticsearch
[root@elk01 ~]# chown -R elasticsearch.elasticsearch /var/log/elasticsearch
[root@elk01 ~]# chown -R elasticsearch.elasticsearch /usr/local/elasticsearch/data
[root@elk01 ~]# ln -s /usr/local/elasticsearch/config /etc/elasticsearch

[root@elk01 ~]# vi /etc/init.d/elasticsearch
#!/bin/sh
#
# elasticsearch <summary>
#
# chkconfig:
# description: Starts and stops a single elasticsearch instance on this system
#

### BEGIN INIT INFO
# Provides: Elasticsearch
# Required-Start: $network $named
# Required-Stop: $network $named
# Default-Start:
# Default-Stop:
# Short-Description: This service manages the elasticsearch daemon
# Description: Elasticsearch is a very scalable, schema-free and high-performance search solution supporting multi-tenancy and near realtime search.
### END INIT INFO

#
# init.d / servicectl compatibility (openSUSE)
#
if [ -f /etc/rc.status ]; then
    . /etc/rc.status
    rc_reset
fi

#
# Source function library.
#
if [ -f /etc/rc.d/init.d/functions ]; then
    . /etc/rc.d/init.d/functions
fi

exec="/usr/local/elasticsearch/bin/elasticsearch"
prog="elasticsearch"
pidfile=/var/run/elasticsearch/${prog}.pid

[ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog

export ES_HEAP_SIZE
export ES_HEAP_NEWSIZE
export ES_DIRECT_SIZE
export ES_JAVA_OPTS

lockfile=/var/lock/subsys/$prog

# backwards compatibility for old config sysconfig files, pre 0.90.
if [ -n $USER ] && [ -z $ES_USER ] ; then
   ES_USER=$USER
fi

checkJava() {
    if [ -x "$JAVA_HOME/bin/java" ]; then
        JAVA="$JAVA_HOME/bin/java"
    else
        JAVA=`which java`
    fi

    if [ ! -x "$JAVA" ]; then
        echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"
        exit
    fi
}

start() {
    checkJava
    [ -x $exec ] || exit
    [ -f $CONF_FILE ] || exit
    if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then
        echo "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set"
        return
    fi
    if [ -n "$MAX_OPEN_FILES" ]; then
        ulimit -n $MAX_OPEN_FILES
    fi
    if [ -n "$MAX_LOCKED_MEMORY" ]; then
        ulimit -l $MAX_LOCKED_MEMORY
    fi
    if [ -n "$MAX_MAP_COUNT" ]; then
        sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
    fi
    if [ -n "$WORK_DIR" ]; then
        mkdir -p "$WORK_DIR"
        chown "$ES_USER":"$ES_GROUP" "$WORK_DIR"
    fi
    echo -n $"Starting $prog: "
    # if not running, start it up here, usually something like "daemon $exec"
    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -D
es.default.path.work=$WORK_DIR -Des.default.path.conf=$CONF_DIR
    retval=$?
    echo
    [ $retval -eq  ] && touch $lockfile
    return $retval
}

stop() {
    echo -n $"Stopping $prog: "
    # stop it here, often "killproc $prog"
    killproc -p $pidfile -d  $prog
    retval=$?
    echo
    [ $retval -eq  ] && rm -f $lockfile
    return $retval
}

restart() {
    stop
    start
}

reload() {
    restart
}

force_reload() {
    restart
}

rh_status() {
    # run checks to determine if the service is running or use generic status
    status -p $pidfile $prog
}

rh_status_q() {
    rh_status >/dev/null >&
}

case "$1" in
    start)
        rh_status_q && exit
        $
        ;;
    stop)
        rh_status_q || exit
        $
        ;;
    restart)
        $
        ;;
    reload)
        rh_status_q || exit
        $
        ;;
    force-reload)
        force_reload
        ;;
    status)
        rh_status
        ;;
    condrestart|try-restart)
        rh_status_q || exit
        restart
        ;;
    *)
        echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
        exit
esac
exit $?

[root@elk01 ~]# chmod +x /etc/init.d/elasticsearch
[root@elk01 ~]# /etc/init.d/elasticsearch start

#管理工具
[root@elk01 ~]# /usr/local/elasticsearch/bin/plugin install lmenezes/elasticsearch-kopf
[root@elk01 ~]# /usr/local/elasticsearch/bin/plugin install mobz/elasticsearch-head

六. Kibana安装

#### Kibana Install ####

[root@elk01 src]# wget https://download.elastic.co/kibana/kibana/kibana-4.2.1-linux-x64.tar.gz
[root@elk01 src]# tar zxf kibana-4.2.-linux-x64.tar.gz -C /usr/local/
[root@elk01 local]# touch /etc/init.d/kibana
[root@elk01 local]# chmod  /etc/init.d/kibana
[root@elk01 local]# vi /etc/init.d/kibana
#!/bin/bash
### BEGIN INIT INFO
# Provides:          kibana
# Default-Start:
# Default-Stop:
# Short-Description: Runs kibana daemon
# Description: Runs the kibana daemon as a non-root user
### END INIT INFO

# Process name
NAME=kibana
DESC="Kibana4"
PROG="/etc/init.d/kibana"

# Configure location of Kibana bin
KIBANA_BIN=/usr/local/kibana/bin

# PID Info
PID_FOLDER=/var/run/kibana/
PID_FILE=/var/run/kibana/$NAME.pid
LOCK_FILE=/var/lock/subsys/$NAME
PATH=/bin:/usr/bin:/sbin:/usr/sbin:$KIBANA_BIN
DAEMON=$KIBANA_BIN/$NAME

# Configure User to run daemon process
DAEMON_USER=root
# Configure logging location
KIBANA_LOG=/var/log/kibana.log

# Begin Script
RETVAL=

if [ `id -u` -ne  ]; then
        echo "You need root privileges to run this script"
        exit
fi

# Function library
. /etc/init.d/functions

start() {
        echo -n "Starting $DESC : "

pid=`pidofproc -p $PID_FILE kibana`
        if [ -n "$pid" ] ; then
                echo "Already running."
                exit
        else
        # Start Daemon
if [ ! -d "$PID_FOLDER" ] ; then
                        mkdir $PID_FOLDER
                fi
daemon --user=$DAEMON_USER --pidfile=$PID_FILE $DAEMON >"$KIBANA_LOG" >& &
                sleep
                pidofproc node > $PID_FILE
                RETVAL=$?
                [[ $? -eq  ]] && success || failure
echo
                [ $RETVAL =  ] && touch $LOCK_FILE
                return $RETVAL
        fi
}

reload()
{
    echo "Reload command is not implemented for this service."
    return $RETVAL
}

stop() {
        echo -n "Stopping $DESC : "
        killproc -p $PID_FILE $DAEMON
        RETVAL=$?
echo
        [ $RETVAL =  ] && rm -f $PID_FILE $LOCK_FILE
}

case "$1" in
  start)
        start
;;
  stop)
        stop
        ;;
  status)
        status -p $PID_FILE $DAEMON
        RETVAL=$?
        ;;
  restart)
        stop
        start
        ;;
  reload)
reload
;;
  *)
# Invalid Arguments, print the following message.
        echo "Usage: $0 {start|stop|status|restart}" >&
exit
        ;;
esac
[root@elk01 local]# mv kibana-4.2.-linux-x64/ kibana
[root@elk01 ~]# mkdir -p /var/run/kibana

七. logstash日志收集

#### logstash 日志收集  ####
[root@localhost ~]#  tar zxf logstash-2.0..tar.gz -C /usr/local/
[root@localhost ~]# /usr/local/logstash-2.0./bin/logstash -e 'input { stdin { } } output { stdout {} }'
[root@localhost logstash-2.0.]# cat logstash_agent.conf
input {
  file {
    type => "apache_access"
    path => ["/var/log/httpd/access_log"]
  }
}

output {
  stdout {codec => rubydebug }
  redis {
    host => '192.168.55.134'
    data_type => 'list'
    key => 'logstash:redis'
  }
}

# 访问一下http服务，看redis是否收到日志
[root@elk01 elasticsearch]# redis-cli monitor
OK
1448364122.959182 [ 127.0.0.1:] "rpush" "logstash:redis" "{\"message\":\"Nov 24 19:22:02 elk01 yum[3074]: Erased: httpd\",\"@version\":\"1\",\"@timestamp\":\"2015-11-24T11:22:02.553Z\",\"host\":\"0.0.0.0\",\"path\":\"/var/log/messages\",\"type\":\"messages\"}"