利用ansible-playbook一键部署ELK(ElasticSearch,logstash and kibana)

一.部署前环境介绍:

es集群5台(es01,es02,es03,es04,es05),logstash服务器1台(logstash2),kibana服务器1台(kibana2),模拟apache服务及filebeat(收集日志工具)1台(web2);以上均由虚拟机模拟实现;

ip分配如下:

192.168.1.11 es01　　

192.168.1.12 es02

192.168.1.13 es03

192.168.1.14 es04

192.168.1.15 es05

192.168.1.21 logstash2

192.168.1.22 kibana2

192.168.1.31 web2

真机:192.168.1.254

通过ftp共享真机yum源在/var/ftp/elk和centos-1804

二.ansible-playbook应用

ansible服务器ip:192.168.1.40

配置ansible:

 echo "[es]
 es01
 es02
 es03
 es04
 es05" >> /etc/ansible/hosts

1.部署脚本elk.yml

 ---
 - name: 环境部署
   hosts: es,logstash2,kibana2,web2
   tasks:
     - name: 环境部署
       script: /root/elk.sh --some-arguments 

 - name: es集群部署
   hosts: es
   tasks:
     - name: 安装jdk,es
       yum:
         name: 'java-1.8.0-openjdk'
         state: latest
     - yum:
         name: 'elasticsearch'
         state: latest
     - name: 修改配置文件
       lineinfile:
         path: /etc/elasticsearch/elasticsearch.yml
         regexp: "{{ item.old }}"
         line: "{{ item.new }}"
       with_items:
         - {old: '# cluster.name',new: 'cluster.name: myelk' }
         - {old: '# network.host',new: 'network.host: 0.0.0.0' }
         - {old: '# discovery.zen.ping.unicast.hosts',new:'discovery.zen.ping.unicast.hosts: ["es01", "es02","es03"]' }
         - {old: '# node.name',new: 'node.name: {{ ansible_nodename }}' }
     - name: reload es
       service:
         name: elasticsearch
         state: restarted
         enabled: yes
 #必须在es部署之后执行
 - name: es01的head和kopf插件安装
   hosts: es01
   tasks:
     - name: 安装head插件
       shell: '/usr/share/elasticsearch/bin/plugin install ftp://192.168.1.254/elk/elasticsearch-head-master.zip'
     - name: 安装kopf插件
       shell: '/usr/share/elasticsearch/bin/plugin install ftp://192.168.1.254/elk/elasticsearch-kopf-master.zip'

 - name: logstash部署
   hosts: logstash2
   tasks:
     - name: 安装jdk,logstash
       yum:
         name: 'java-1.8.0-openjdk'
         state: latest
     - yum:
         name: 'logstash'
         state: latest
     - name: 方便apache日志读取
       script: /root/elk2.sh --some-arguments 

 - name: kibana部署
   hosts: kibana2
   tasks:
     - name: 安装kibana
       yum:
         name: 'kibana'
         state: latest
     - name: 修改配置文件
       lineinfile:
         path: /opt/kibana/config/kibana.yml
         regexp: "{{ item.old2 }}"
         line: "{{ item.new2 }}"
       with_items:
         - {old2: 'server.port',new2: ' server.port: 5601' }
         - {old2: 'server.host',new2: ' server.host: "0.0.0.0"' }
         - {old2: 'elasticsearch.url',new2: ' elasticsearch.url: "http://192.168.1.11:9200"' }
         - {old2: 'kibana.index',new2: ' kibana.index: ".kibana"' }
         - {old2: 'kibana.defaultAppId',new2: ' kibana.defaultAppId: "discover"' }
         - {old2: 'elasticsearch.pingTimeout',new2: ' elasticsearch.pingTimeout: 1500' }
         - {old2: 'elasticsearch.requestTimeout',new2: ' elasticsearch.requestTimeout: 30000' }
         - {old2: 'elasticsearch.startupTimeout',new2: ' elasticsearch.startupTimeout: 5000' }
     - name: reload kibana
       service:
         name: kibana
         state: restarted
         enabled: yes

 - name: web服务和filebeat部署
   hosts: web2
   tasks:
     - name: 安装apache,filebeat
       yum:
         name: 'httpd'
         state: latest
     - yum:
         name: 'filebeat'
         state: latest
     - name: 修改配置文件
       lineinfile:
         path: /etc/filebeat/filebeat.yml
         regexp: "{{ item.old3 }}"
         line: "{{ item.new3 }}"
       with_items:
         - {old3: 'elasticsearch:',new3: '#  elasticsearch:' }
         - {old3: 'localhost:9200"',new3: '#hosts: ["localhost:9200"]' }
         - {old3: '#logstash:',new3: '  logstash:' }
         - {old3: 'localhost:5044"',new3: '    hosts: ["192.168.1.21:5044"]' }
     - replace:
         path: /etc/filebeat/filebeat.yml
         regexp: '{{ item.old4 }}'
         replace: '{{ item.new4 }}'
         backup: yes
       with_items:
         - {old4: '\*\.',new4: 'access_' }
     - name: reload http,filebeat
       service:
         name: 'httpd'
         state: restarted
         enabled: yes
     - service:
         name: 'filebeat'
         state: restarted
         enabled: yes

2.调用的shell脚本

/root/elk.sh

 #!/bin/bash
 echo "127.0.0.1       localhost localhost.localdomain localhost4 localhost4.localdomain4
 192.168.1.11 es01
 192.168.1.12 es02
 192.168.1.13 es03
 192.168.1.14 es04
 192.168.1.15 es05
 192.168.1.21 logstash2
 192.168.1.22 kibana2" > /etc/hosts
 mkdir /var/ftp/elk
 echo "[local_repo]
 name=CentOS-$releasever - Base
 baseurl="ftp://192.168.1.254/centos-1804"
 enabled=
 gpgcheck=
 [elk]
 name=elk
 baseurl="ftp://192.168.1.254/elk"
 enabled=
 gpgcheck=
 " > /etc/yum.repos.d/local.repo #elasticsearch,logstash,kibana,filebeat安装包
 yum clean all
 yum repolist

/root/elk2.sh

 #!/bin/bash
 touch /etc/logstash/logstash.conf
 echo 'input{
         stdin{codec => "json"}
         beats{
                 port =>
         }
         file{
                 path => ["/tmp/c.log"]
                 type => "test"
                 start_position => "beginning"
                 sincedb_path => "/var/lib/logstash/sincedb"
         }
 }
 filter{
         if [type] == "apache_log"{
         grok{
                 match => {"message" => "%{COMBINEDAPACHELOG}"}
         }}
 }
 output{
         stdout{ codec => "rubydebug" }
         if [type] == "apache_log"{
         elasticsearch{
                 hosts => ["192.168.1.51:9200","192.168.1.52:9200"]
                 index => "filelog"
                 flush_size =>
                 idle_flush_time =>
         }}
 }
 ' > /etc/logstash/logstash.conf