ELK + Redis 日志收集 & HAProxy

参考博文：http://www.ttlsa.com/linux/haproxy-log-configuration-syslog/

引入 Redis 消息队列

Log-file 收集数据到 Redis

主机	IP	部署服务
web01	172.16.1.7	Nginx，Tomcat，Logstash
dbtest01	172.16.1.121	ElasticSearch，Kibana，Redis
dbtest02	172.16.1.122	ElasticSearch
dbtest03	172.16.1.123	ElasticSearch

收集 Nginx & Tomcat 日志

# 日志文件到 Redis
[root@web01 ~]# vim /etc/logstash/conf.d/file_to_redis.conf
input {
  file {
    type => "nginx_log"
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
    codec => "json"
  }
  file {
    type => "tomcat_log"
    path => "/usr/local/tomcat/logs/localhost_access_log.*.log"
    start_position => "beginning"
    codec => "json"
  }
}
output {
  if [type] == "nginx_log" {
    redis {
      host => "172.16.1.121"
      port => "6379"
      data_type => "list"
      db => "0"
      key => "nginx_log"
    }
  }
  if [type] == "tomcat_log" {
    redis {
      host => "172.16.1.121"
      port => "6379"
      data_type => "list"
      db => "0"
      key => "tomcat_log"
    }
  }
}

#=============== 或者（简写）================#
# 日志文件到 Redis
[root@web01 conf.d]# cat file_to_redis.conf
input {
  file {
    type => "nginx_log"
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
    codec => "json"
  }
  file {
    type => "tomcat_log"
    path => "/usr/local/tomcat/logs/localhost_access_log.*.log"
    start_position => "beginning"
    codec => "json"
  }
}

output {
  redis {
    host => ["172.16.1.121"]
    port => "6379"
    data_type => "list"
    db => "0"
    key => "%{type}"
  }
}

# 验证：访问 Nginx 和 Tomcat 页面，查看 Redis 里面有没有 Key
127.0.0.1:6379> LLEN nginx_log
(integer) 1
127.0.0.1:6379> LLEN nginx_log
(integer) 888
127.0.0.1:6379> LRANGE nginx_log 0 -1

Redis 收集数据到 ElasticSearch

# Redis 到 es
[root@web01 conf.d]# cat redis_to_es.conf
input {
  redis {
    host => "172.16.1.121"
    port => "6379"
    db => "0"
    data_type => "list"
    key => "nginx_log"
  }
  redis {
    host => "172.16.1.121"
    port => "6379"
    db => "0"
    data_type => "list"
    key => "tomcat_log"
  }
}

output {
  if [type] == "nginx_log" {
    elasticsearch {
      hosts => ["10.0.0.121:9200"]
      index => "nginx_log_%{+YYYY-MM-dd}"
    }
  }
  if [type] == "tomcat_log" {
    elasticsearch {
      hosts => ["10.0.0.121:9200"]
      index => "tomcat_log_%{+YYYY-MM-dd}"
    }
  }
}

启动 Logstash 多实例

[root@web01 conf.d]# mkdir /data/logstash/file_to_redis
[root@web01 conf.d]# mkdir /data/logstash/redis_to_es
[root@web01 conf.d]# chown logstash.logstash /data -R

[root@web01 conf.d]# logstash -f /etc/logstash/conf.d/file_to_redis.conf --path.data=/data/logstash/file_to_redis &

[root@web01 conf.d]# logstash -f /etc/logstash/conf.d/redis_to_es.conf --path.data=/data/logstash/redis_to_es &

TCP / UDP 模块

TCP 模块初识

[root@web01 ~]# vim /etc/logstash/conf.d/tcp.conf
input {
  tcp {
    port => "1234"
    mode => "server"
  }
}
output {
  stdout {}
}

使用 telnet 工具测试

[root@db02 ~]# telnet 172.16.1.7 1234
Trying 172.16.1.7...
Connected to 172.16.1.7.
Escape character is '^]'.
123
345

# 输出内容
{
    "@timestamp" => 2020-08-17T02:23:05.833Z,
          "host" => "172.16.1.52",
          "port" => 33002,
       "message" => "\r",
      "@version" => "1"
}
{
    "@timestamp" => 2020-08-17T02:23:32.562Z,
          "host" => "172.16.1.52",
          "port" => 33002,
       "message" => "123\r",
      "@version" => "1"
}
{
    "@timestamp" => 2020-08-17T02:23:38.300Z,
          "host" => "172.16.1.52",
          "port" => 33002,
       "message" => "345\r",
      "@version" => "1"
}

使用 nc 工具测试

# 安装
[root@db02 ~]# yum install -y nc

# 使用 nc 工具
[root@db02 ~]# nc 172.16.1.7 1234
123
456

# 使用 nc 工具收集日志到 logstash 的服务器
[root@web01 ~]# tail -f /var/log/nginx/access.log | nc 10.0.0.7 1234 &
[1] 29595

# 发送伪设备数据
[root@web01 ~]# echo "伪设备测试" > /dev/tcp/10.0.0.7/1234

收集日志到 ElasticSearch

[root@web01 ~]# vim /etc/logstash/conf.d/tcp.conf
input {
  tcp {
    port => "1234"
    mode => "server"
  }
}
output {
  elasticsearch {
    hosts => ["10.0.0.121:9200"]
    index => "tcp_log_%{+YYYY-MM-dd}"
  }
}

Rsyslog + Logstash 收集日志

Rsyslog 介绍

Rsyslog 是一个快速处理收集系统日志的程序，提供了高性能、安全功能和模块化设计，Rsyslog 是 Syslog 的升级版，它将多种来源输入输出转换结果到目的地，据官网介绍，现在可以处理 100 万条信息

Rsyslog 安装

[root@web01 ~]# yum isntall -y rsyslog

Rsyslog 配置

[root@web01 ~]# vim /etc/rsyslog.conf
# 打开注释
$ModLoad imudp
$UDPServerRun 514
$ModLoad imtcp
$InputTCPServerRun 514
# 添加日志收集级别
local6.*       @@172.16.1.7:2222

安装 HAProxy

[root@web01 ~]# yum install -y haproxy

配置 HAProxy

[root@web01 ~]# cat /etc/haproxy/haproxy.cfg
# 全局配置
global
# 最大并发
maxconn 100000
# 安全机制
chroot /var/lib/haproxy
# 指定启动的用户和组
uid 99
gid 99
daemon
# haproxy 的进程数
nbproc 1
pidfile /var/run/haproxy.pid
# 指定日志级别
log 127.0.0.1 local6 info

# 默认配置
defaults
# 开启长连接
option http-keep-alive
# 获取用户真实 IP
option  forwardfor
# 最大连接数
maxconn 100000
# 支持 http 协议
mode http
# 设置连接超时时间
timeout connect 300000ms
timeout client  300000ms
timeout server  300000ms

# 监控状态
listen stats
 # 支持 http
 mode http
 # 监听端口
 bind 0.0.0.0:9999
 # 启动
 stats enable
 # 日志级别
 log global
 # 访问 uri 地址
 stats uri     /haproxy-status
 # 状态页用户名和密码
 stats auth    haadmin:123456

#frontend web_port
frontend web_port
        bind 0.0.0.0:80
        mode http
        option httplog
        log global
        option  forwardfor
###################ACL Setting##########################
        acl nginx       hdr_dom(host) -i www.nginx.com
        acl tomcat      hdr_dom(host) -i www.tomcat.com
###################USE ACL##############################
        use_backend     nginx_host     if  nginx
        use_backend     tomcat_host    if  tomcat
########################################################

backend nginx_host
        mode    http
        option  httplog
        balance source
        server web01  10.0.0.7:8081 check inter 2000 rise 3 fall 2 weight 1

backend tomcat_host
        mode    http
        option  httplog
        balance source
        server web01  10.0.0.7:8080 check inter 2000 rise 3 fall 2 weight 1

修改 Nginx 启动端口

[root@web01 ~]# vim /etc/nginx/nginx.conf
    server {
        listen       8081 default_server;
        ...

启动 HAProxy

# 启动 haproxy
[root@web01 ~]# systemctl start haproxy.service

# 启动 rsyslog，用来收集 haproxy 日志，转发到 172.16.1.7:2222 端口
[root@web01 ~]# systemctl start rsyslog

# 验证，rsyslog 开启 514 端口， haproxy 开启 80 端口（frontend）& 9999 端口（stats）
[root@web01 ~]# netstat -lntp

访问 Status 页面

# 访问 http://10.0.0.7:9999/haproxy-status
# 用户：haadmin
# 密码：123456

访问 Nginx 和 Tomcat

# 配置本地 hosts
10.0.0.7 www.nginx.com
10.0.0.7 www.tomcat.com

# 访问页面
http://www.nginx.com/
http://www.tomcat.com/

Logstash 收集 HAProxy 日志到 Stdout

[root@web01 ~]# cat /etc/logstash/conf.d/haproxy.conf
input {
  syslog {
    port => "2222"
  }
}
output {
  stdout {}
}

# 访问 haproxy 的页面，查看有无输出

Logstash 收集 HAProxy 日志到 ElasticSearch

[root@web01 ~]# cat /etc/logstash/conf.d/haproxy.conf
input {
  syslog {
    port => "2222"
  }
}
output {
  elasticsearch {
    hosts => ["10.0.0.121:9200"]
    index => "haproxy_%{+YYYY-MM-dd}"
  }
}