logstash收集nginx访问日志

安装nginx

#直接yum安装：

[root@elk-node1 ~]# yum install nginx  -y

官方文档：http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format

#修改配置文件的日志格式：

vim /etc/nginx/nginx.conf

#在http模块中添加

          log_format json '{"@timestamp":"$time_iso8601",'

                           '"@version":"1",'

                           '"client":"$remote_addr",'

                           '"url":"$uri",'

                           '"status":"$status",'

                           '"domain":"$host",'

                           '"host":"$server_addr",'

                           '"size":$body_bytes_sent,'

                           '"responsetime":$request_time,'

                           '"referer": "$http_referer",'

                           '"ua": "$http_user_agent"'

               '}';

#在server模块中添加

access_log  /var/log/nginx/access_json.log  json;

#修改后的nginx.conf文件

[root@elk-node1 ~]# grep -Ev "#|^&" /etc/nginx/nginx.conf

user nginx;

worker_processes auto;

error_log /var/log/nginx/error.log;

pid /run/nginx.pid;

include /usr/share/nginx/modules/*.conf;

events {

    worker_connections 1024;

}

http {

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '

                      '$status $body_bytes_sent "$http_referer" '

                      '"$http_user_agent" "$http_x_forwarded_for"';

          log_format json '{"@timestamp":"$time_iso8601",'

                           '"@version":"1",'

                           '"client":"$remote_addr",'

                           '"url":"$uri",'

                           '"status":"$status",'

                           '"domain":"$host",'

                           '"host":"$server_addr",'

                           '"size":$body_bytes_sent,'

                           '"responsetime":$request_time,'

                           '"referer": "$http_referer",'

                           '"ua": "$http_user_agent"'

               '}';

    sendfile            on;

    tcp_nopush          on;

    tcp_nodelay         on;

    keepalive_timeout   65;

    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;

    default_type        application/octet-stream;

    include /etc/nginx/conf.d/*.conf;

    server {

        listen       80 default_server;

        listen       [::]:80 default_server;

        server_name  _;

        root         /usr/share/nginx/html;

        include /etc/nginx/default.d/*.conf;

    access_log  /var/log/nginx/access_json.log  json;

        location / {

        }

        error_page 404 /404.html;

            location = /40x.html {

        }

        error_page 500 502 503 504 /50x.html;

            location = /50x.html {

        }

    }

}

#启动：

[root@controller ~]# systemctl start nginx

root@elk-node1 ~]# ss -lntp|grep 80

LISTEN     0      511          *:80                       *:*                   users:(("nginx",pid=8045,fd=6),("nginx",pid=8044,fd=6),("nginx",pid=8043,fd=6))

LISTEN     0      511         :::80                      :::*                   users:(("nginx",pid=8045,fd=7),("nginx",pid=8044,fd=7),("nginx",pid=8043,fd=7))

浏览器访问：http://192.168.247.135/

查看nginx日志

编写logstash

#添加nginx日志格式到之前logstash的elk-log.yml

[root@elk-node1 ~]# cat /etc/logstash/conf.d/elk_log.conf

input {

    file {

      path => "/var/log/messages"

      type => "system"

      start_position => "beginning"

    }

    file {

       path => "/var/log/elasticsearch/hejianlai.log"

       type => "es-error"

       start_position => "beginning"

      codec => multiline {

          pattern => "^\["

          negate => true

          what => "previous"

        }

    }

       file {

       path => "/var/log/nginx/access_json.log"

       codec => json

       start_position => "beginning"

       type => "nginx-log"

    }

}

output {

    if [type] == "system"{

        elasticsearch {

           hosts => ["192.168.247.135:9200"]

           index => "systemlog-%{+YYYY.MM.dd}"

        }

    }

    if [type] == "es-error"{

        elasticsearch {

           hosts => ["192.168.247.135:9200"]

           index => "es-error-%{+YYYY.MM.dd}"

        }

    }

       if [type] == "nginx-log"{

        elasticsearch {

           hosts => ["192.168.247.135:9200"]

           index => "nginx-log-%{+YYYY.MM.dd}"

        }

    }

}

#添加--configtest参数检查配置语法是否有误！！！
[root@elk-node1 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf --configtest
Configuration OK
#把之前后台运行的进程kill掉重启：
[root@elk-node1 ~]# ps aux|grep elk
root 3248 0.8 6.0 3632960 234924 pts/2 Sl 11:25 1:10 /usr/local/java/jdk1.8.0_171/bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -Xmx1g -Xss2048k -Djffi.boot.library.path=/opt/logstash/vendor/jruby/lib/jni -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/logstash/heapdump.hprof -Xbootclasspath/a:/opt/logstash/vendor/jruby/lib/jruby.jar -classpath ::/usr/local/java/jdk1.8.0_171/lib:/usr/local/java/jdk1.8.0_171/jre/lib -Djruby.home=/opt/logstash/vendor/jruby -Djruby.lib=/opt/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main --1.9 /opt/logstash/lib/bootstrap/environment.rb logstash/runner.rb agent -f /etc/logstash/conf.d/elk_log.conf
root 8135 0.0 0.0 112704 976 pts/0 S+ 13:38 0:00 grep --color=auto elk
[root@elk-node1 ~]# kill -9 3248
You have new mail in /var/spool/mail/root
[root@elk-node1 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf &
[1] 8178

kibana添加nginx日志

首先在es插件中我们能看到nginx-log的索引

设置kibana