logstash/conf.d文件编写

logstash-01.conf

input {

beats {
port => 5044
host => "0.0.0.0"
type => "logs"
codec => "json"
}

}

filter{

if ( [type] == "nginx-access" ) {

grok {
match => {
"request" => "\s+(?<api_path>.+?)(\?.*)?\s+"
}
}
grok {
match => {
"agent" => "(?<browser>Maxthon|QQBrowser|Chrome|Safari|Firefox|Opera|MSIE?)(/[0-9.]+)?"
}
}
grok {
match => {
"agent" => "(?<os>Macintosh|Android|SymbianOS|iPad|iPhone|iPod|Linux|Windows?)"
}
}
mutate {
split => [ "upstreamtime", "," ]
}

mutate {
remove_field => [ "offset", "@version", "beat", "input_type", "tags", "id" ]
}

date {
match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
}

geoip{
source => "clientip" # 取自nginx中的客户端ip
target => "geoip"
database => "/usr/share/logstash/geoip/GeoLite2-City.mmdb"
}
mutate {
convert => [ "[geoip][location]", "float" ]
convert => [ "[responsetime]", "float" ]
convert => [ "[upstreamtime]", "float" ]
convert => [ "[size]", "float" ]
convert => [ "[status]", "float" ]
convert => [ "[cookie]", "float" ]
}

}
if ( [type] == "nginx-error" ) {
grok {
match => [ "message" , "(?<timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:severity}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(?:, client: (?<remote_addr>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server}?)(?:, request: %{QS:request})?(?:, upstream: (?<upstream>\"%{URI}\"|%{QS}))?(?:, host: %{QS:request_host})?(?:, referrer: \"%{URI:referrer}\")?"]
}
}

}
output {

stdout {
codec => rubydebug { }
}

if [type] == "nginx-access" {
#redis {
# host => "127.0.0.1"
# port => "6379"
# password => '123.com'
# db => '2'
# data_type => "list"
# key => 'nginx-access'
# }

elasticsearch {
hosts => ["192.168.10.196:9200"]
index => "logstash-nginx-access-%{+YYYY.MM.dd}"
#codec => "json"
template_overwrite => true
}
}
if [type] == "nginx-error" {

elasticsearch {
hosts => ["192.168.10.196:9200"]
index => "logstash-nginx-error-%{+YYYY.MM.dd}"
codec => "json"
template_overwrite => true
}
}
}

logstash-java-02.conf

input {
redis {
host => "192.168.10.196"
port => 6379
password => "123.com"
db => "1"
data_type => "list"
key => "java-access"
codec => "json"
threads => 5
}
stdin{
codec => multiline {
#pattern => "^%{LOGLEVEL} "
pattern => "^%{TIMESTAMP_ISO8601} "
negate => true
what => "previous"
}
}

}

filter{

if ( [type] == "java-access" ) {
grok{
match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{JAVALOGMESSAGE:msg}" }
}

date {
match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss,S", "ISO8601" ]
}
json {
source => "message"
}
}
}
output {

stdout {
codec => rubydebug { }
}

if [type] == "java-access" {
elasticsearch {
hosts => ["192.168.10.196:9200"]
index => "logstash-java-access-%{+YYYY.MM.dd}"
template_overwrite => true
}
}
}

logstash-oms-04.conf

input {
redis {
host => "192.168.10.196"
port => 6379
db => "0"
data_type => "list"
password => "123.com"
key => "oms-access"
codec => "json"
threads => 5
}

}

filter{
if ( [type] == "java-oms" ) {
grok{
match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{JAVALOGMESSAGE:msg}" }
}

date {
match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss,S", "ISO8601" ]
}
json {
source => "message"
}
}

}
output {

stdout {
codec => rubydebug { }
}

if [type] == "java-oms" {
elasticsearch {
hosts => ["192.168.10.196:9200"]
index => "logstash-java-oms-%{+YYYY.MM.dd}"
template_overwrite => true
}
}
}

logstash-payment-03.conf

input {
redis {
host => "192.168.10.196"
port => 6379
db => "1"
data_type => "list"
password => "123.com"
key => "java-access"
codec => "json"
threads => 5
}

}

filter{
if ( [type] == "java-payment" ) {
grok{
match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{JAVALOGMESSAGE:msg}" }
}

date {
match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss,S", "ISO8601" ]
}
json {
source => "message"
}
}
}
output {

stdout {
codec => rubydebug { }
}

if [type] == "java-payment" {
elasticsearch {
hosts => ["192.168.10.196:9200"]
index => "logstash-java-payment-%{+YYYY.MM.dd}"
template_overwrite => true
}
}
}