Redis做消息队列

1.连接从Redis中获取日志文件并存储到ES中

[root@Logstash ~]# vim /usr/local/logstash/config/redis.conf

input {

    beats {

    port => "5044"

     }

    redis {

    data_type => "list"

        key => "220"

        host => "192.168.200.134"

        port => 6379

        db => 0

        threads => 1

    }

}

filter {

    if [type] == "nginx"{

    grok {

    match => { "message" => "%{NGINXACCESS}" }

 

  }

 

  }

 

  date {

    match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]

    target => ["datetime"]

  }

  geoip {

    source => "clientip"

  }

 

}

 

output {

 if [fields][logsource] == "220nginx_access"{

     elasticsearch {

      hosts => ["192.168.200.130:9200"]

      index => "220nginx_access"

     }

}

   if [fields][logsource] == "220nginx_error"{

     elasticsearch {

      hosts => ["192.168.200.130:9200"]

      index => "220nginx_error"

     }

  }

 

        stdout { codec => rubydebug }

}

2.将Nginx访问日志和错误日志推送到Redis消息队列中。

[root@mobanji filebeat]# egrep -v "#|^$" filebeat.yml

filebeat.prospectors:

- type: log

  fields:

    logsource: 220nginx_access

    log_type: access_log

    fields_under_root: true

  paths:

    - /usr/local/nginx/logs/access.log

- type: log

  fields:

    logsource: 220nginx_error

    log_type: error_log

  paths:

    - /usr/local/nginx/logs/error.log

output.redis:

  hosts: ["192.168.200.134"]

  port: 6379

  db: 0

  timeout: 5

  key: "220"

3.收集Nginx和Tomcat日志：

[root@mobanji ~]# egrep -v "#|^$" /usr/local/filebeat/filebeat.yml

filebeat.prospectors:

- type: log

  fields:

    logsource: 220nginx_access

    log_type: access_log

    fields_under_root: true

  paths:

    - /usr/local/nginx/logs/access.log

- type: log

  fields:

    logsource: 220nginx_error

    log_type: error_log

  paths:

    - /usr/local/nginx/logs/error.log

- type: log

  fields:

    logsource: 220tomcat_out

    log_type: tomcat_out

  paths:

    - /usr/local/tomcat/logs/catalina.out

output.redis:

  hosts: ["192.168.200.134"]

  port: 6379

  db: 0

  timeout: 5

  key: "220"