docker方式部署elk日志搜索平台

Docker部署ELKF操作文档

前提介绍

1、之前搭建elk+f+k使用原生系统软件安装方式，由于docker镜像日趋成熟，docker官网和elastic官网都有相关镜像和各自安装文档可供参考，各个版本也在定期更新，这次决定换用docker方式进行搭建安装。

Docker（elk）的hub网站链接及文档：

https://hub.docker.com/r/sebp/elk

https://elk-docker.readthedocs.io/

2、搭建前准备：

[由于公司资源有限，开一台虚拟机放置elk所有插件，若有足够资源，可考虑使用分布式部署及es集群方式]

空余主机一台（内存>=6G）：

　　Linux Centos7.6

用到的主要软件有：

Elasticsearch7.0.0 (搜索引擎-server端)

Kibana7.0.0 (图形化web界面-server端)

Logstash7.0.0 (log的汇总与收集-server端)

Filebeats7.0.1 (log收集-client端) 或 Metricbeat-7.0.1(client端)

一、安装docker及elk相关软件

yum list | grep docker

yum makecache fast

###移除旧版本###

yum remove docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-engine

yum install yum-utils device-mapper-persistent-data lvm2 -y

ls /etc/yum.repos.d/

yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo

ls /etc/yum.repos.d/docker-ce.repo

###只开启需要安装项###

yum-config-manager --disable docker-ce-nightly

yum-config-manager --disable docker-ce-test

yum-config-manager --enable docker-ce

yum install docker-ce docker-ce-cli containerd.io

systemctl status docker

systemctl start docker

systemctl enable docker

systemctl status docker

docker –version

docker info

###调整系统内核###

sysctl -w vm.max_map_count=262144

###搜索相关镜像###

docker search elk

docker search sebp/elk

docker pull sebp/elk:700

echo $?

docker ps

docker rmi b0e0bd2a140b #删除镜像

docker images

docker rm 30fe8fb61024 #删除容器

docker ps -a

docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -v /home/elk-data:/elk-data -it --name elk sebp/elk:700 –d（后台守护启动）

docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -v /home/elk-data:/elk-data -it -d --name elk sebp/elk:700

netstat -anptu|grep 5601

netstat -anptu|grep 9200

###进入后台运行容器###

docker exec -it elk /bin/bash

exit退出

二、采集端安装Filebeat

（这里是用的原生软件安装）

#curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.0.1-linux-x86_64.tar.gz

tar -zxvf filebeat-7.0.1-linux-x86_64.tar.gz

mv filebeat-7.0.1-linux-x86_64 /home/

cd /home/

mv filebeat-7.0.1-linux-x86_64/ filebeat7.0

cd filebeat7.0/

cat filebeat.yml | grep -vE '^$|#'

#filebeat配置，主要收集nginx、php和php-slow日志

filebeat.inputs:

- type: log

paths:

- /home/logs/hao.log

fields:

software: nginx

logname: hao

servername: "server-81"

fields_under_root: true

- type: log

paths:

- /home/logs/zf.error.log

fields:

software: nginx-error

logname: zf-error

servername: "server-81"

fields_under_root: true

- type: log

paths:

- /usr/local/php/var/log/php-fpm7.log

fields:

software: php

logname: php-fpm

servername: "server-81"

fields_under_root: true

- type: log

paths:

- /usr/local/php/var/log/slow.log

multiline.pattern: '^\[[0-9]{2}'

multiline.negate: true

multiline.match: after

fields:

software: php

logname: php-slow

servername: "server-81"

fields_under_root: true

#默认

filebeat.config.modules:

path: ${path.config}/modules.d/*.yml

reload.enabled: false

setup.template.settings:

index.number_of_shards: 1

setup.kibana:

#数据输出到logstash[此ip为公司外网映射到内网elk的ip]

output.logstash:

hosts: ["222.111.11.22:5044"]

#默认

processors:

- add_host_metadata: ~

- add_cloud_metadata: ~

###启动filebeat###

./filebeat -c filebeat.yml &

ps -ef|grep filebeat

netstat -anptu |grep 5044

###统一nginx日志格式，其他php采用正则匹配###

vim /etc/nginx/nginx.conf #主配置格式#子配置access … main；

…

log_format main escape=json

'{"nx_localtime@timestamp":"$time_local",'

'"nx_host":"$server_addr",'

'"nx_client_ip":"$remote_addr",'

'"nx_body_size":$body_bytes_sent,'

'"nx_request_time":$request_time,'

'"nx_scheme":"$scheme",'

'"nx_http_host":"$host",'

'"nx_request_method":"$request_method",'

'"nx_uri":"$uri",'

'"nx_status":$status,'

'"nx_referer":"$http_referer",'

'"nx_agent":"$http_user_agent",'

'"nx_upstream_host":"$upstream_addr",'

'"nx_upstream_time":$upstream_response_time,' '"nx_upstream_response_length":$upstream_response_length,'

'"nx_upstream_status":$upstream_status,'

'"nx_upstream_connect_time":"$upstream_connect_time"}';

…

三、容器内部操作及配置文件

#docker exec –it elk /bin/bash

ls /elk-data/

cd opt/logstash/config/

cp logstash-sample.conf logstash.conf

mv /etc/logstash/*.conf /etc/logstash/*.conf.bak

vim logstash.conf

# Sample Logstash configuration for creating a simple

# Beats -> Logstash -> Elasticsearch pipeline.

input {

beats {

codec => json

port => 5044

host => "0.0.0.0"

client_inactivity_timeout => 300

ssl => false

}

filter {

if [software] == "nginx-error" {

grok {

match => { "message" => "(?<timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:level}\] %{POSINT:pid}#%{NUMBER}: %{DATA:code} %{GREEDYDATA:event}(?:, client: (?<client_ip>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server_website}?)(?:, request: \"%{WORD:http_method})? (?:%{NOTSPACE:request_uri})? (?:%{NOTSPACE:http_version}\")?(?:, upstream: (?<upstream_ori>\"%{URI}\"|%{QS}))?(?:, host: %{QS:request_host})?(?:, referrer: \"%{URI:referrer_host}\")?" }

remove_field => ["code","[prospector][type]"]

}

if [software] == "php" and [logname] == "php-fpm" {

grok {

match => [ "message","\[%{TIMESTAMP_ISO8601:logtime}\] %{WORD:env}\.(?<level>[A-Z]{4,5})\: %{GREEDYDATA:msg}}" ]

}

if [software] == "php" and [logname] == "php-slow" {

grok {

match => { "message" => "^\[%{DATA:time_local}\].*?script_filename\s+=\s+%{DATA:script_filename}\n%{GREEDYDATA:msg}" }

}

mutate {

remove_field => ["@version","tags","[ecs][version]","[agent][hostname]","[agent][ephemeral_id]","[agent][id]","[agent][type]","[agent][version]","[host][containerized]","[host][hostname]","[host][os][codename]","[host][os][family]","[host][os][platform]"]

#remove_field => "message"

}

- output {

#输出nginx的访问日志

if [software] == "nginx" {

elasticsearch {