filebeat+elasticsearch+kibana

一、到elasticsearch官网下载 filebeat+elasticsearch+kibana

http://www.elasticsearch.cn/

二、安装filebeat

tar -xzvf filebeat-7.16.3-linux-x86_64.tar.gz -C /opt
cd /opt
mv filebeat-7.16.3-linux-x86_64 filebeat
groupadd -g 1004 fbeat
useradd -u 1004 fbeat -g 1004 -s /sbin/nologin -M
chown -R fbeat.fbeat filebeat

systemd纳管filebeat

cat <<efo> /usr/lib/systemd/system/filebeat.service
[Unit]
Description=filebeat 7.16.3
After=syslog.target network.target docker.service

[Service]
#Type=simple
User=fbeat
Group=fbeat
ExecStart=/opt/filebeat/filebeat -c /opt/filebeat/filebeat.yml
PrivateTmp=true

[Install]
WantedBy=multi-user.target

efo

systemctl daemon-reload

systemctl enable filebeat --now

filebeat.yml模板

filebeat.inputs:
- type: log
  enabled: true
  paths:
  #日志所在的路径
    - /usr/local/nginx/logs/*.log
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
  multiline.negate: true
  multiline.match: after
  #给每个日志做个标识
  fields:
     source: index_name
  processors:
  - drop_fields:
      fields: ["container","host", "tags", "ecs", "prospector", "agent", "input", "beat", "offset","kubernetes"]
      ignore_missing: true
  clean_*: 48h
  close_*: 5m
  idle_timeout: 30s
  scan_frequency: 8s
filebeat.config.modules:
  # Glob pattern for configuration loading
  path: ${path.config}/modules.d/*.yml
  # Set to true to enable config reloading
  reload.enabled: false
  # Period on which files under path should be checked for changes
  #reload.period: 10s
#es配置的是集群，所以这里配置3
setup.template.settings:
  index.number_of_shards: 3
  index.number_of_replicas: 1
  #index.codec: best_compression
  #_source.enabled: false
setup.template.enabled: true
setup.template.overwrite: true
setup.template.name: "index_name-"
setup.template.pattern: "index_name-*"
setup.ilm.enabled: false
setup.kibana:
  host: ""           #kibana地址
output.elasticsearch:
  enabled: true
  hosts: [""]       #es地址
  indices:
    - index: "index_name-day-%{+yyyy.MM.dd}"     #索引模式名称
      when.equals:
        fields.source: "index_name"          #根据上面做的日志标识，创建相应日期的索引
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
  - add_docker_metadata: ~
  - add_kubernetes_metadata: ~

三、安装es

vi /etc/sysctl.conf
行尾增加:  vm.max_map_count=655360
sysctl -p

groupadd elasticsearch
useradd -m -g elasticsearch elasticsearch
chown -R elasticsearch /elasticsearch
chmod 755 -R /elasticsearch/config/
chmod 777 -R /elasticsearch/logs/
chmod 777 -R /elasticsearch/data/

配置 elasticsearch.yml

vi elasticsearch.yml
cluster.name: elk-test
node.name: elk-test-node1
network.host: 0.0.0.0
network.publish_host: 节点IP
http.port: 9200
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: Authorization,X-Requested-With,Content-Length,Content-Type
discovery.seed_hosts: ["多个节点IP以逗号分隔"] #集群成员
cluster.initial_master_nodes: ["主节点IP"] #指定主节点
node.master: true
node.data: true

#启动es
docker run -d --name elasticsearch --restart=always  -p 9200:9200 -p 9300:9300 -v /elasticsearch/config:/usr/share/elasticsearch/config -v /elasticsearch/data:/usr/share/elasticsearch/data -v /elasticsearch/logs:/usr/share/elasticsearch/logs  elasticsearch:7.16.3

四、安装kibana

配置 kibana.yml

server.publicBaseUrl: "http://IP:5601"
server.host: "0.0.0.0"
server.name: "node1"
server.port: 5601
elasticsearch.hosts: ["http://es地址:9200"]
i18n.locale: "zh-CN"
monitoring.ui.container.elasticsearch.enabled: true

#启动kibana
docker run --name=kibana -p 5601:5601 --restart=always -v/kibana/config:/usr/share/kibana/config -d kibana:7.16.3