logstash配置

input {
#You must define a [type], otherwise you cannot get a field to cut.
tcp {
port => 5045
type => "iis_mail_log"
codec => "json"
#start_position => "beginning"
}

#Configure syslog type,Collect Fortigate Firewall log
syslog {
port => 514
type => "syslog_net"
}

#Configure syslog type,the same to up.
#tcp {
# port => 5140
# type => syslog
#}

#udp {
# port => 5140
# type => "syslog"
#}

}

filter {
if [type] in ["iis_mail_log","iis_oa_log"]{
geoip {
source => "c-ip"
target => "geoip"
database => "/data/app/logstash-6.2.2/data/GeoLite2-City.mmdb"
fields => ["city_name","region_name","country_name"]
}
}

#Filter ldap log
if [type] == "sec_ldap_log" {
json {
source => "message"
}
if [SourceModuleName] == "seclogas" {
mutate {
replace => [ "message", "%{Message}" ]
}
mutate {
remove_field => [ "Message" ]
}
}

}

# Remove IPv6 prefix from IPAddress if not used
if [IpAddress] =~ "ffff" {
grok {
match => ["IpAddress", "^.*?\::ffff:%{GREEDYDATA:IpAddress}$"]
overwrite => ["IpAddress"]
}
}

#Identify machine accounts
if [TargetUserName] =~ /\$/ {
mutate {
add_field => { "machine" => "true" }
}
} else {
mutate {
add_field => { "machine" => "false" }

}
}

# # Extract username from email
# if [TargetUserName] =~ /\@/ {
# grok {
# match => ["TargetUserName", "%{WORD:TargetUserName}"]
# overwrite => ["TargetUserName"]
# }
# }

# Filter Fortigate firewall log
if [type] == "syslog_net" {

grok {
match => ["message","<%{POSINT:syslog_index}>%{GREEDYDATA:message}"]
overwrite => ["message"]
}

kv {
source => "message"
field_split => ","
value_split => "="
trim_value => "\""
include_keys => ["date","time","subtype","srcip","srcport","srcintf","dstip","dstport","dstintf","action","trandisp","transip","service"]
#target => "kv"
}

mutate{
add_field => ["fg_time","%{date} %{time}"]
remove_field => ["date","time"]
# rename => ["type","fg_type"]
# rename => ["subtype","fg_subtype"]
# add_field => ["type" => "syslog_net"]
#convert => ["rcvdbyte" => "interger"]
#convert => ["sentbyte" => "integer"]

}
#date {
# match => ["temp_time","yyyy-MM-dd HH:mm:ss"]
# # timezone => "UTC"
# target => "@timestamp"
#}

}

}

output {
if [type] == "iis_mail_log" {
#Output to redis
redis {
host => ["2.2.2.2:6379"]
key => "logstash"
data_type => "channel"
codec => "json"
}
Output to elasticasearch
elasticsearch {
action => "index"
hosts => ["1.1.2.1:9200","1.1.2.2:9200"]
index => "iis_mail_%{+YYYY-MM}"
codec => "json"
}
}

if [type] == "iis_oa_log"{
elasticsearch {
action => "index"
hosts => ["1.1.2.1:9200","1.1.2.2:9200"]
#index => "logstash-oa-access0529-%{+YYYY-MM}"
index => "iis_oa_%{+YYYY-MM}"
codec => "json"
}
}

if [type] == "syslog_net"{
elasticsearch {
action => "index"
hosts => ["1.1.2.1:9200","1.1.2.2:9200"]
index => "net_fw_%{+YYYY-MM}"
codec => "json"
}
}

}