logstash-安装

1.下载

 

cd /usr/local/src

 

wget https://mirrors.huaweicloud.com/logstash/7.8.0/logstash-7.8.0.tar.gz

 

tar -zxvf logstash-7.8.0.tar.gz

 

2.启动测试

 

执行命令:

 

bin/logstash -e 'input { stdin { } } output { stdout {} }'

 

 

输出:

 

[2020-09-23T10:09:21,198][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}

hello

{

    "@timestamp" => 2020-09-23T14:09:25.158Z,

       "message" => "hello",

      "@version" => "1",

          "host" => “localhost.localdomain"

 

 

 

3.json输出

 

执行命令:

 

bin/logstash -e 'input { stdin {} } output { stdout { codec => json } }'

 

输出:

 

[2020-09-23T10:15:44,836][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}

hello

{"@timestamp":"2020-09-23T14:16:15.777Z","host":"localhost.localdomain","message":"hello","@version":"1”}

 

 

4.把命令放在文件中使用

 

vim test.conf

 

input {

  stdin { }

}

 

output {

  stdout { }

}

 

 

执行命令:

 

bin/logstash -f test.conf

 

输出:

 

[2020-09-23T10:22:12,515][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}

hello

/usr/local/src/logstash-7.8.0/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated

{

       "message" => "hello",

      "@version" => "1",

          "host" => "localhost.localdomain",

    "@timestamp" => 2020-09-23T14:22:39.016Z

}

 

 

 

5.监控指定文件

 

vim test1.conf

 

input {

  file { path => "/tmp/hello.log"  }

}

 

output {

  stdout { }

}

 

//或者

 

input {

  file { path => "/tmp/hello.log"  }

}

 

 

output {

  stdout {

  codec => "json"

}

}

 

 

 

 

执行命令:

 

 

bin/logstash -f test1.conf

 

 

另外开一个窗口 输入日志到文件

 

 

[root@localhost logstash-7.8.0]# echo hello >> /tmp/hello.log

[root@localhost logstash-7.8.0]# echo world >> /tmp/hello.log

 

 

 

输出:

 

 

[2020-09-23T10:27:29,766][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}

/usr/local/src/logstash-7.8.0/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated

{

          "host" => "localhost.localdomain",

    "@timestamp" => 2020-09-23T14:27:44.789Z,

       "message" => "hello",

      "@version" => "1",

          "path" => "/tmp/hello.log"

}

{

          "host" => "localhost.localdomain",

    "@timestamp" => 2020-09-23T14:27:58.948Z,

       "message" => "world",

      "@version" => "1",

          "path" => "/tmp/hello.log"

}

 

 

6.监控文件输出到elasticsearch

 

input {

  file{ path => "/deng/log/host.log"

        type => "log"

        start_position => "beginning"

       }

}

 

 

output {

  elasticsearch {

     hosts => ["127.0.0.1:9200"]

     index => “ml"

    }

}

 

执行命令:

 

bin/logstash -f test.conf

 

 

7.过滤器插件 grok

 

vim test3.conf

 

 

input {

        file { path => "/deng/log/host.log" }

}

filter {

      grok { 

match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes}"  } 

} 

}

 

output { stdout { } }  

 

 

 

 

 

 

 

另外开一个窗口 输入日志到文件:

 

[root@localhost logstash-7.8.0]# echo "55.3.244.1 GET /index.html 15824" >> /tmp/hello.log

 

输出:

 

{

        "client" => "55.3.244.1",

    "@timestamp" => 2020-09-23T14:45:44.770Z,

      "@version" => "1",

        "method" => "GET",

       "request" => "/index.html",

          "path" => "/tmp/hello.log",

          "host" => "localhost.localdomain",

         "bytes" => "15824",

       "message" => "55.3.244.1 GET /index.html 15824"

}

 

 

8.使用remove_field去掉message这一行的信息

 

filter{

  mutate {

    remove_field => ["message","timestamp"]

  }

}

 

 

9.添加字段

 

  mutate {

    split => ["message", "|"]

      add_field => {

        "timestamp" => "%{[message][0]}"

　　　}

　}

}