elasticsearch5.5安装

beat --------  kafka -------- logstash---------------elasticsearch--------------kibana

beat配置文件

[root@varnish1 filebeat-5.5.-linux-x86_64]# grep -v "#" filebeat.yml  | sed '/^$/d'
filebeat.prospectors:
- input_type: log
  paths:
    - /data/*.log
tags: ["haproxy-log"]
output.kafka:
  enabled : true
  hosts: ["kafka1:9092","kafka2:9092","kafka3:9092"]
  topic: logostash-%{[type]}
[root@varnish1 filebeat-5.5.0-li

logstash配置文件

[root@logstashserver etc]# cat logstash.conf
input {
        kafka {
        bootstrap_servers => "kafka1:9092,kafka2:9092,kafka3:9092"
        topics => ["logostash-log"]
        consumer_threads =>
        decorate_events => true
            }
}

filter {
    grok{
       patterns_dir => "/data/logstash/patterns"
       match => {"message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{FENG:captured_request_cookie} %{FENG:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} \"%{WORD:verb} %{URIPATHPARAM:request} %{WORD:http_socke}/%{NUMBER:http_version}\""}
       }
    geoip {
        source => "client_ip"
        fields => ["ip","city_name","country_name","location"]
        add_tag => [ "geoip" ]
        }
}

output {
           elasticsearch {
           hosts => ["es1:9200","es2:9200","es3:9200"]
     manage_template => true
           index => "logstash-feng.log-%{+YYYY-MM-dd}"
    }
}

在es1上操作：

下载elasticsearch5.5安装：

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.5.0.tar.gz

解压：

tar -zxvf elasticsearch-5.5.0.tar.gz

mv elasticsearch-5.5.0   elasticsearch

接下来创建el用户，因为elasticsearch不允许以root运行（其实也可以运行，需要配置）。

1.设置虚拟 –Xms  -Xmx 内存大小

vim  /data/elasticsearch/config/jvm.options

2.设置虚拟内存

echo “vm.max_map_count=262144” >> /etc/sysctl.conf

3.关闭swap分区

Swapoff –a

修改配置文件fstab

#/dev/mapper/centos-swap swap           swap    defaults        0 0

4.设置elastrisearch用户名密码

useradd es

passwd es

5. 需要修改
/etc/security/limits.conf 
es soft memlock unlimited
es hard memlock unlimited

6.修改：
/etc/sysctl.conf 
vm.swappiness=0

之后重启机器

chown -R es:es /data/elasticsearch

su - es

cd elasticsearch/conf/

修改配置文件：

vim elasticsearch.yml

cluster.name: senyint_elasticsearch
node.name: es1
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/logs
bootstrap.memory_lock: false
network.host: 192.168.99.8
http.port:
discovery.zen.ping.unicast.hosts: ["es1", "es2", "es3"]
http.cors.enabled: true
http.cors.allow-origin: "*"
xpack.security.enabled: false     #安装x-pack 插件使用

安装X-Pack

在5.x版本中一些es插件(Shield, Watcher, Marvel, Graph, Reporting)都集成在x-pack组件中

在Es的根目录(每个节点),运行 bin/elasticsearch-plugin进行安装

bin/elasticsearch-plugin install x-pack

安装head插件：

#elasticsearch-head 不能放置在 /data/elasticsearch/plugins/目录下，否者报错

cd /data/elasticsearch/

git clone git://github.com/mobz/elasticsearch-head.git

root用户

cd /data/

wget https://nodejs.org/dist/v6.11.1/node-v6.11.1-linux-x64.tar.xz

xz -d node-v6.11.1-linux-x64.tar.xz

tar -xf node-v6.11.1-linux-x64.tar

vi /etc/profile

export NODE_HOME=/data/node-v6.11.1-linux-x64

export PATH=${PATH}:${NODE_HOME}/bin

source /etc/profile

su -  es

cd /data/elasticsearch/elasticsearch-head/

npm install

vim /data/elasticsearch/elasticsearch-head/Gruntfile.js

vim /data/elasticsearch/elasticsearch-head/_site/app.js

启动elasticsearch

su - es

/data/elasticsearch/bin/elasticsearch  -d

启动 grunt

cd   /data/elasticsearch/plugins/elasticsearch-head/node_modules/grunt/bin

#  nohup ./grunt  server  &

访问 http://192.168.99.9:9100

安装kibana

Kibana是一个强大的es图形化组件,可以通过http的方式来查看es集群的状态,操作数据等.

当前es官方推荐的拓展组件为Kibana和X-Pack,而X-Pack需要依赖于Kibana.

下载地址：https://artifacts.elastic.co/downloads/kibana/kibana-5.5.0-linux-x86_64.tar.gz

选择任意一台es节点安装kibana

vim config/kibana.yml
server.port:
server.host: "192.168.10.116"
elasticsearch.url: "http://192.168.99.8:9200"
xpack.security.enabled: false 

在Kibana根目录运行 bin/kibana-plugin 进行安装

bin/kibana-plugin install x-pack

启动Kibana

启动之前需要禁用X-Pack 插件 security

vim config/kibana.yml

vim config/elasticsearch.yml

添加以下内容

xpack.security.enabled: false     #重要

nginx 日志格式

    log_format logstash_json '{ "@timestamp": "$time_local", '
                         '"client": "$remote_addr", '
                         '"hostname": "$hostname", '
                         '"remote_user": "$remote_user", '
                         '"upstream_addr": "$upstream_addr", '
                         '"upstream_response_time": "$upstream_response_time", '
                         '"body_bytes_sent": "$body_bytes_sent", '
                         '"request_time": "$request_time", '
                         '"status": "$status", '
                         '"request": "$request", '
                         '"request_method": "$request_method", '
                         '"http_referrer": "$http_referer", '
                         '"body_bytes_sent":"$body_bytes_sent", '
                         '"http_x_forwarded_for": "$http_x_forwarded_for", '
                         '"http_user_agent": "$http_user_agent"  }';

    access_log  logs/access.log  logstash_json;

filebeat 部署在nginx服务器上，收集日志

filebeat.prospectors:
- input_type: log
  paths:
    - /data/nginx/logs/access.log
  document_type: nginx_access
output.kafka:
  enabled : true
  hosts: ["kafka1:9092","kafka2:9092","kafka3:9092"]
  topic: logostash_%{[type]}

logstash 从kafka订阅消息，并存储到es中配置文件

input {
        kafka {
                bootstrap_servers => "kafka1:9092,kafka2:9092,kafka3:9092"
                topics => ["logostash__nginx_access"]
                consumer_threads =>
                decorate_events => true
                codec => "json"
                }
}

filter {
    json {
        remove_field => ["@timestamp","beat","type","kafka","domain","serverip","url","@version","offset","input_type","count","source","fields","beat.hostname","host","tags"]
        source => "message"
        remove_field => ["message"]
    }

}

output {
        elasticsearch {
        hosts => ["es1:9200","es2:9200","es3:9200"]
        manage_template => true
        index => "logstash-nginx.log-%{+YYYY-MM-dd}"
        }
}