第七章·Logstash深入-收集NGINX日志

1.NGINX安装配置

源码安装nginx

因为资源问题，我们先将nginx安装在Logstash所在机器

#安装nginx依赖包
[root@elkstack03 ~]# yum install -y gcc gcc-c++ automake pcre-devel zlib-devel openssl-devel
#下载nginx安装包
[root@elkstack03 ~]# wget http://nginx.org/download/nginx-1.10.3.tar.gz
#解压
[root@elkstack03 ~]# tar xf nginx-1.10.3.tar.gz
#进入nginx安装目录
[root@elkstack03 ~]# cd nginx-1.10.3/
#生成编译文件
[root@elkstack03 nginx-1.10.3]#  ./configure  --prefix=/usr/local/nginx-1.10.3
#编译
[root@elkstack03 nginx-1.10.3]# make
#安装
[root@elkstack03 nginx-1.10.3]# make install
#做软链接
[root@elkstack03 nginx-1.10.3]# ln -s /usr/local/nginx-1.10.3 /usr/local/nginx
#检测nginx语法
[root@elkstack03 nginx-1.10.3]# /usr/local/nginx/sbin/nginx -t
nginx: the configuration file /usr/local/nginx-1.10.3/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx-1.10.3/conf/nginx.conf test is successful
#启动nginx
[root@elkstack03 nginx-1.10.3]# /usr/local/nginx/sbin/nginx

---

配置nginx

#简化nginx配置文件
[root@elkstack03 ~]# grep -Ev '#|^$' /usr/local/nginx/conf/nginx.conf.default  > /usr/local/nginx/conf/nginx.conf
#编辑nginx配置文件
[root@elkstack03 ~]# vim /usr/local/nginx/conf/nginx.conf
worker_processes  1;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
    keepalive_timeout  65;
    server {
        listen       80;
        server_name  localhost;
        location / {
            root   /code/html;
            index  index.html index.htm;
        }
    }
}
#创建nginx站点目录
[root@elkstack03 ~]# mkdir /code/html
#写测试页面
[root@elkstack03 ~]# echo zls nginx test page > /code/html/index.html
#重新加载nginx
[root@elkstack03 ~]# /usr/local/nginx/sbin/nginx -s reload

打开浏览器，访问：http://10.0.0.53/


---

修改nginx日志格式为Json

之前我们讲了tomcat日志，在企业中，修改格式需要与开发商量，但是nginx我们不需要，如果需要原来的格式日志，我们可以将日志输出两份，一份 main格式，一份Json格式

#编辑nginx日志，添加日志格式，源main格式和Json格式
[root@elkstack03 ~]# vim /usr/local/nginx/conf/nginx.conf
worker_processes  1;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
    keepalive_timeout  65;

#main格式日志
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  logs/access.log  main;

#Json格式日志
    log_format access_json '{"@timestamp":"$time_iso8601",'
            '"host":"$server_addr",'
            '"clientip":"$remote_addr",'
            '"size":$body_bytes_sent,'
            '"responsetime":$request_time,'
            '"upstreamtime":"$upstream_response_time",'
            '"upstreamhost":"$upstream_addr",'
            '"http_host":"$host",'
            '"url":"$uri",'
            '"domain":"$host",'
            '"xff":"$http_x_forwarded_for",'
            '"referer":"$http_referer",'
            '"status":"$status"}';
    access_log  logs/access_json.log  access_json;

    server {
        listen       80;
        server_name  10.0.0.53;
        location / {
            root   /code/html;
            index  index.html index.htm;
        }
    }
}
#检测nginx配置文件语法
[root@elkstack03 ~]# /usr/local/nginx/sbin/nginx -t
nginx: the configuration file /usr/local/nginx-1.10.3/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx-1.10.3/conf/nginx.conf test is successful
#重新加载nginx
[root@elkstack03 ~]# /usr/local/nginx/sbin/nginx -s reload

打开浏览器，访问：http://10.0.0.53/ 查看日志

#进入nginx日志目录
[root@elkstack03 ~]# cd /usr/local/nginx/logs/
#查看目录中日志
[root@elkstack03 logs]# ll
总用量 24
#修改后的Json格式日志
-rw-r--r-- 1 root root 1280 4月   8 10:47 access_json.log
#源main格式日志
-rw-r--r-- 1 root root 5286 4月   8 10:47 access.log
-rw-r--r-- 1 root root 4218 4月   8 10:46 error.log
-rw-r--r-- 1 root root    5 4月   8 10:20 nginx.pid

#查看Json格式日志
[root@elkstack03 logs]# cat access_json.log
{"@timestamp":"2019-04-08T10:47:41+08:00","host":"10.0.0.53","clientip":"10.0.0.1","size":0,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"10.0.0.53","url":"/index.html","domain":"10.0.0.53","xff":"-","referer":"-","status":"304"}

#查看main格式日志
[root@elkstack03 logs]# cat access.log
10.0.0.1 - - [08/Apr/2019:10:29:11 +0800] "GET / HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"

结果如下：



---

通过Logstash收集nginx日志输出到ES中

[root@elkstack03 ~]# cd /etc/logstash/conf.d/
[root@elkstack03 conf.d]# vim nginx_es.conf
input {
  file {
    path => "/usr/local/nginx/logs/access_json.log"
    start_position => "end"
    type => "nginx_access"
    codec => json
  }
}

output {
    elasticsearch {
      hosts => ["10.0.0.51:9200"]
      index => "nginx_access-%{+YYYY.MM.dd}"
   }
}
#检测Logstash语法
[root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_es.conf -t
#启动Logstash
[root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_es.conf &

打开浏览器，访问：http://10.0.0.51:9100/



---

将ES中的索引添加到Kibana中

打开浏览器，访问：http://10.0.0.54:5601/ Kibana页面