elk-（七）

最终架构确定为  logs--->blieb--->redis/kafka--->logstash--->es--->kibana

注意：  geoip下载地址：

wget     http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz

filebeat  配置文件

filebeat-nginx

filebeat.prospectors:
- type: log
  paths:
    - /usr/local/nginx/logs/access.log              #nginx路径
  # tags: ["access"]
  fields:
    app: www                                        #所属于的项目
    type: nginx-access
  fields_under_root: true                           #目录制顶

- type: log
  paths:
    - /usr/local/nginx/logs/error.log
  # ags: ["error"]
  fields:
    app: www
    type: nginx-error
  fields_under_root: true

output.redis:                                     #输入的redis
  hosts: ["192.168.0.215"]                        #ip  密码  key  数据库 数据类型
  password: ""
  key: "filebeat"
  db: 0
  datatype: list

filebeat-tomcat

filebeat.prospectors:
- type: log
  paths:
    - /usr/local/tomcat/logs/catalina.out
  # tags: ["tomcat"]
  fields:
    app: www
    type: tomcat-catalina
  fields_under_root: true
  multiline:
    pattern: '^\['           #自定义正则
    negate: true
    match: after

output.redis:
  hosts: ["192.168.0.215"]
  password: ""
  key: "filebeat"
  db: 0
  datatype: list

nginx-json格式化

log_format json '{ "@timestamp": "$time_iso8601", '
         '"remote_addr": "$remote_addr", '
         '"remote_user": "$remote_user", '
         '"body_bytes_sent": "$body_bytes_sent", '
         '"request_time": "$request_time", '
         '"status": "$status", '
         '"request_uri": "$request_uri", '
         '"request_method": "$request_method", '
         '"http_referrer": "$http_referer", '
         '"http_x_forwarded_for": "$http_x_forwarded_for", '
         '"http_user_agent": "$http_user_agent"}';

logstash配置文件

logstash-to-es-nginxjson.com    （nginx json格式化日志）

input {  #进入的类型
    redis {
        host => "192.168.0.215"
        port => 6379
        password => ""
        db => ""
        data_type => "list"
        key => "filebeat"
    }
}

filter {
  if [app] == "www" {    #判断项目
    if [type] == "nginx-access" {  #判断类型
      json {
          source => "message"
          remove_field => ["message"]
      }
      geoip {
          source => "remote_addr"
          target => "geoip"
          database => "/opt/GeoLite2-City.mmdb"   #geoip  数据库地址  可以自定义
          add_field => ["[geoip][coordinates]", "%{[geoip][longitude]}"]
          add_field => ["[geoip][coordinates]", "%{[geoip][latitude]}"]
      }
      mutate {
          convert => ["[geoip][coordinates]", "float"]
      }
    }
  }
}

output {
  elasticsearch {
      hosts  => ["http://192.168.0.212:9200","http://192.168.0.213:9200","http://192.168.0.214:9200"]   #es集群
      index  => "logstash-%{type}-%{+YYYY.MM.dd}"
  }
  stdout{codec => rubydebug }
}

logstash-to-es-custom.conf    自定义格式化

input {
    redis {
        host => "192.168.0.215"
        port => 6379
        password => ""
        db => ""
        data_type => "list"
        key => "filebeat"
    }
}

filter {
  if [app] == "www" {
    if [type] == "nginx-access" {
      grok {
        match => {
          "message" => "%{IPV4:remote_addr} - (%{USERNAME:remote_user}|-) \[%{HTTPDATE:time_local}\] \"%{WORD:request_method} %{URIPATHPARAM:request_uri} HTTP/%{NUMBER:http_protocol}\" %{NUMBER:http_status} %{NUMBER:body_bytes_sent} \"%{GREEDYDATA:http_referer}\" \"%{GREEDYDATA:http_user_agent}\" \"(%{IPV4:http_x_forwarded_for}|-)\""
        }   #自定义格式化
        overwrite => ["message"]
      }
      geoip {
          source => "remote_addr"
          target => "geoip"
          database => "/opt/GeoLite2-City.mmdb"
          add_field => ["[geoip][coordinates]", "%{[geoip][longitude]}"]
          add_field => ["[geoip][coordinates]", "%{[geoip][latitude]}"]
      }
      date {
          locale => "en"
          match => ["time_local", "dd/MMM/yyyy:HH:mm:ss Z"]
      }
      mutate {
          convert => ["[geoip][coordinates]", "float"]
      }
    }
  }
}

output {
  elasticsearch {
      hosts  => ["http://192.168.0.212:9200","http://192.168.0.213:9200","http://192.168.0.214:9200"]
      index  => "logstash-%{type}-%{+YYYY.MM.dd}"
  }
  stdout{codec => rubydebug }
}