ELKStack入门篇（二）之Nginx、Tomcat、Java日志收集以及TCP收集日志使用

1、收集Nginx的json格式日志

1.1、Nginx安装

[root@linux-node1 ~]# yum install nginx -y

[root@linux-node1 ~]# vim /etc/nginx/nginx.conf

#修改日志格式为json格式，并创建一个nginxweb的网站目录

log_format access_json '{"@timestamp":"$time_iso8601",'

                           '"host":"$server_addr",'

                           '"clientip":"$remote_addr",'

                           '"size":$body_bytes_sent,'

                           '"responsetime":$request_time,'

                           '"upstreamtime":"$upstream_response_time",'

                           '"upstreamhost":"$upstream_addr",'

                           '"http_host":"$host",'

                           '"url":"$uri",'

                           '"domain":"$host",'

                           '"xff":"$http_x_forwarded_for",'

                           '"referer":"$http_referer",'

                           '"status":"$status"}';

    access_log  /var/log/nginx/access.log  access_json;

        location /nginxweb {

                root html;

                index index.html index.htm;

        }

[root@linux-node1 ~]# mkdir /usr/share/nginx/html/nginxweb

[root@linux-node1 ~]# echo "<h1> welcome to use Nginx" > /usr/share/nginx/html/nginxweb/index.html

[root@linux-node1 ~]# nginx -t

nginx: the configuration file /etc/nginx/nginx.conf syntax is ok

nginx: configuration file /etc/nginx/nginx.conf test is successful

[root@linux-node1 ~]# systemctl start nginx

1.2、配置logstash

[root@linux-node1 ~]# vim /etc/logstash/conf.d/nginx-accesslog.conf

input{

    file {

        path => "/var/log/nginx/access.log"

        type => "nginx-access-log"

        start_position => "beginning"

        stat_interval => ""

    }

}

output{

   elasticsearch {

        hosts => ["192.168.56.11:9200"]

        index => "logstash-nginx-access-log-%{+YYYY.MM.dd}"

   }

   file {

        path => "/tmp/logstash-nginx-access-log-%{+YYYY.MM.dd}"

   }

}

[root@linux-node1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_access.conf -t

[root@linux-node1 ~]# systemctl restart logstash

1.3、配置Kibana展示

[root@linux-node1 ~]# ab -n1000 -c  http://192.168.56.11/nginxweb/index.html    #对页面压测

[root@linux-node1 ~]# tailf /var/log/nginx/access.log    #nginx的访问日志变成了json格式

{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}

{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}

{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}

{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}

{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}

{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}

{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}

{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}

{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}

{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}

Head插件查看：

Kibana查看：

2、Tomcat的json日志收集

2.1、下载tomcat

[root@linux-node2 ~]# wget http://apache.fayea.com/tomcat/tomcat-8/v8.5.20/bin/apache-tomcat-8.5.20.tar.gz

[root@linux-node2 ~]# tar -zxf apache-tomcat-8.5..tar.gz

[root@linux-node2 ~]# mv apache-tomcat-8.5. /usr/local/tomcat

2.2、修改tomcat日志格式

[root@linux-node2 ~]# cd /usr/local/tomcat/conf

[root@linux-node2 conf ]# cp server.xml{,.bak}

[root@linux-node2 conf ]# vim server.xml

<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"

               prefix="tomcat_access_log" suffix=".log"

               pattern="{"clientip":"%h","ClientUser":"%l","authenticated":"%u","AccessTime":"%t","method":"%r","status":"%s","SendBytes":"%b","Query?string":"%q","partner":"%{Referer}i","AgentVersion":"%{User-Agent}i"}"/>

2.3、增加tomcat网页

[root@linux-node2 conf ]# cd ../webapps/

[root@linux-node2 webapps ]# mkdir webdir && cd webdir

[root@linux-node2 webdir ]# echo "<h1>welcome to use tomcat</h1>" > index.html

[root@linux-node2 conf ]# ../bin/catalina.sh start

[root@linux-node2 conf ]# netstat -tulnp |grep

tcp6              :::                 :::*                    LISTEN      /java

2.4、压测页面，生成tomcat的访问日志

[root@linux-node2 conf ]# ab -n1000 -c100 http://192.168.56.12:8080/webdir/index.html

[root@linux-node2 ~]# tailf /usr/local/tomcat/logs/tomcat_access_log.--.log

{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}

{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}

{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}

{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}

{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}

{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}

{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}

{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:56 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}

{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}

{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}

2.5、配置logstash

[root@linux-node2 ~]# vim /etc/logstash/conf.d/tomcat_accesslog.conf

input {

        file {

                path => "/usr/local/tomcat/logs/tomcat_access_log.*.log"

                type => "tomcat-accesslog"

                start_position => "beginning"

                stat_interval => ""

        }

}

output {

        elasticsearch {

                hosts => ["192.168.56.11:9200"]

                index => "logstash-tomcat5612-accesslog-%{+YYYY.MM.dd}"

        }

        file {

                path => "/tmp/logstash-tomcat5612-accesslog-%{+YYYY.MM.dd}"

        }

}

2.6、检查logstash的配置语法并重启logstash

[root@linux-node2 tomcat]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tomcat_accesslog.conf -t

OpenJDK -Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N

WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults

Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console

Configuration OK

[root@linux-node2 tomcat]# systemctl restart logstash

[root@linux-node2 tomcat]# ps -ef |grep logstash

logstash         : ?        :: /bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction= -XX:+UseCMSInitiatingOccupancyOnly -XX:+DisableExplicitGC -Djava.awt.headless=true -Dfile.encoding=UTF- -XX:+HeapDumpOnOutOfMemoryError -Xmx1g -Xms256m -Xss2048k -Djffi.boot.library.path=/usr/share/logstash/vendor/jruby/lib/jni -Xbootclasspath/a:/usr/share/logstash/vendor/jruby/lib/jruby.jar -classpath : -Djruby.home=/usr/share/logstash/vendor/jruby -Djruby.lib=/usr/share/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main /usr/share/logstash/lib/bootstrap/environment.rb logstash/runner.rb --path.settings /etc/logstash

root          : pts/    :: grep --color=auto logstash

2.7、elasticsearch的head插件查看

数据浏览：

注：此处如果无法出现tomcat的数据索引，查看一下是否有权限访问日志。

[root@linux-node2 logs]# ll /usr/local/tomcat/logs/ -d

drwxr-x---  root root  12月  : /usr/local/tomcat/logs/

[root@linux-node2 logs]# chmod  /usr/local/tomcat/logs

[root@linux-node2 logs]# ll /usr/local/tomcat/logs/

总用量

-rw-r-----  root root    12月  : catalina.--.log

-rw-r-----  root root    12月  : catalina.out

-rw-r-----  root root       12月  : host-manager.--.log

-rw-r-----  root root     12月  : localhost.--.log

-rw-r-----  root root       12月  : manager.--.log

-rw-r-----  root root  12月  : tomcat_access_log.--.log

[root@linux-node2 logs]# chmod  /usr/local/tomcat/logs/

2.8、添加到Kibana

3、Java日志收集

使用codec的multiline插件实现多行匹配，这是一个可以将多行进行合并的插件，而且可以使用what指定将匹配到的行与前面的行合并还是和后面的行合并。
https://www.elastic.co/guide/en/logstash/6.0/plugins-codecs-multiline.html

语法例子：

input {

  stdin {

    codec => multiline {    #使用multiline插件

      pattern => "pattern, a regexp"   #正则匹配

      negate => "true" or "false"     #匹配是否成功

      what => "previous" or "next"   #和上面的还是和下面的内容合并

    }

  }

}

命令行测试输入输出

[root@linux-node1 ~]# /usr/share/logstash/bin/logstash -e 'input { stdin {codec => multiline { pattern => "^\[" negate => "true" what => "previous"} }} output { stdout {codec => rubydebug}}'

OpenJDK -Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N

WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults

Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console

The stdin plugin is now waiting for input:

[

{

      "@version" => "",

          "host" => "linux-node1",

    "@timestamp" => --28T03::.663Z,

       "message" => "111111111\n22222222222\n333333333",   #会将[开头前面的进行合并

          "tags" => [

        [] "multiline"

    ]

}

[

{

      "@version" => "",

          "host" => "linux-node1",

    "@timestamp" => --28T03::.326Z,

       "message" => "[5555555555\n666666666666666666\n77777777777777777\n8888888888",

          "tags" => [

        [] "multiline"

    ]

}

3.1、举例

（1）查看elk集群日志

elk集群日志上都是以”[“开头并且每一个信息都是如此，寻找规律

[root@linux-node1 ~]# tailf /data/logs/elk-cluster.log

[--28T09::,][INFO ][o.e.c.s.MasterService    ] [elk-node1] zen-disco-node-join[{elk-node2}{CcF5fl9sRqCAGYYpT3scuw}{ncgZ1UsPRq-iz6zWHPl7PQ}{192.168.56.12}{192.168.56.12:}], reason: added {{elk-node2}{CcF5fl9sRqCAGYYpT3scuw}{ncgZ1UsPRq-iz6zWHPl7PQ}{192.168.56.12}{192.168.56.12:},}

[--28T09::,][INFO ][o.e.c.s.ClusterApplierService] [elk-node1] added {{elk-node2}{CcF5fl9sRqCAGYYpT3scuw}{ncgZ1UsPRq-iz6zWHPl7PQ}{192.168.56.12}{192.168.56.12:},}, reason: apply cluster state (from master [master {elk-node1}{Ulw9eIPlS06sl8Z6zQ_z4g}{HgJRMEAcQcqFOTn5ehHPdw}{192.168.56.11}{192.168.56.11:} committed version [] source [zen-disco-node-join[{elk-node2}{CcF5fl9sRqCAGYYpT3scuw}{ncgZ1UsPRq-iz6zWHPl7PQ}{192.168.56.12}{192.168.56.12:}]]])

[--28T09::,][WARN ][o.e.d.z.ElectMasterService] [elk-node1] value for setting "discovery.zen.minimum_master_nodes" is too low. This can result in data loss! Please set it to at least a quorum of master-eligible nodes (current value: [-], total number of master-eligible nodes used for publishing in this round: [])

[--28T09::,][INFO ][o.e.c.r.a.AllocationService] [elk-node1] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[.kibana][]] ...]).

[--28T09::,][INFO ][o.e.c.m.MetaDataCreateIndexService] [elk-node1] [logstash-tomcat5612-accesslog-2017.12.] creating index, cause [auto(bulk api)], templates [logstash], shards []/[], mappings [_default_]

[--28T09::,][INFO ][o.e.c.m.MetaDataMappingService] [elk-node1] [logstash-tomcat5612-accesslog-2017.12./YY4yqUQJRHa2mRUwmd2Y8g] create_mapping [tomcat-accesslog]

[--28T09::,][INFO ][o.e.c.r.a.AllocationService] [elk-node1] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[logstash-tomcat5612-accesslog-2017.12.][]] ...]).

（2）配置logstash

[root@linux-node1 ~]# vim /etc/logstash/conf.d/java.conf

input {

        file{

                path => "/data/logs/elk-cluster.log"

                type => "elasticsearch-java-log"

                start_position => "beginning"

                stat_interval => ""

                code => multiline {

                        pattern => "^\["    #以"["开头进行正则匹配，匹配规则

                        negate => "true"  #正则匹配成功，false匹配不成功

                        what => "previous"  #和前面的内容进行合并,如果是和下面的合并就是next

                }

        }

}

output {

        if [type] == "elasticsearch-java-log" {

                elasticsearch {

                        hosts => ["192.168.56.11:9200"]

                        index => "elasticsearch-jva-log-%{+YYYY.MM.dd}"

                }

        }

}

[root@linux-node1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/java.conf -t

OpenJDK -Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N

WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults

Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console

Configuration OK

[root@linux-node1 ~]# systemctl restart logstash

（3）elasticsearch的head插件查看