通过logstash收集mysql慢查询日志转换为json

input {
file {
type => "mysql-slow"
path => "/var/log/slow_mysqld.log"
start_position => "beginning"
codec => multiline {
pattern => "^# Time:"
negate => true
what => "previous"
}
}
}
filter {
grok {
match => { "message" => "SELECT SLEEP" }
add_tag => [ "sleep_drop" ]
tag_on_failure => []
}
if "sleep_drop" in [tags] {
drop {}
}
grok {
match => [ "message", "(?m)^# Time:.*\s+# User@Host: %{USER:user}\[[^\]]+\] @ (?:(?<clienthost>\S*) )?\[(?:%{IP:clientip})?\]\s*Id: %{NUMBER:id}\s+# Query_time: %{NUMBER:query_time}\s+Lock_time: %{NUMBER:lock_time}\s+Rows_sent: %{NUMBER:rows_sent}\s+Rows_examined: %{NUMBER:rows_examined}\s*(?:use %{DATA:database};\s*)?SET timestamp=%{NUMBER:timestamp};\s*(?<query>(?<action>\w+)\s+.*)$" ]
}
grok {
match => [ "message", "(?m)^# Time: %{GREEDYDATA:Time}\s+# User@Host: %{USER:user}\[[^\]]+\] @ (?:(?<clienthost>\S*) )?\[(?:%{IP:clientip})?\]\s*Id: %{NUMBER:id}\s+# Schema: %{GREEDYDATA:schema}\s+# Query_time: %{NUMBER:query_time}\s+Lock_time: %{NUMBER:lock_time}\s+Rows_sent: %{NUMBER:rows_sent}\s+Rows_examined: %{NUMBER:rows_examined}\s+Rows_affected: %{NUMBER:Rows_affected}\s+# Bytes_sent: %{NUMBER:Bytes_sent}\s+SET timestamp=%{NUMBER:sqltimestamp};\s*(?<query>(?<action>\w+)\s+.*)$" ]
}
date {
match => [ "timestamp", "UNIX" ]
remove_field => [ "timestamp" ]
}
}
output {
     stdout {
 codec => rubydebug {}
}
}

日志格式

# Time: 2018-11-22T01:58:21.726750Z
# User@Host: xxx[xxx] @  [xx.xx.xx.xx]  Id: 23258299
# Schema: xxx  Last_errno: 0  Killed: 0
# Query_time: 3.533099  Lock_time: 0.000021  Rows_sent: 2  Rows_examined: 2517002  Rows_affected: 0
# Bytes_sent: 975
SET timestamp=1542851901;
SELECT * FROM `x`  WHERE `xxx`
# Time: 2018-11-22T01:58:21.726750Z
# User@Host: xxx[xxx] @  [xx.xx.xx.xx]  Id: 23258299
# Schema: xxx  Last_errno: 0  Killed: 0
# Query_time: 3.533099  Lock_time: 0.000021  Rows_sent: 2  Rows_examined: 2517002  Rows_affected: 0
# Bytes_sent: 975
SET timestamp=1542851901;
SELECT * FROM `x`  WHERE `xxx`
# Time: 2018-11-22T01:58:21.726750Z
# User@Host: xxx[xxx] @  [xx.xx.xx.xx]  Id: 23258299
# Schema: xxx  Last_errno: 0  Killed: 0
# Query_time: 3.533099  Lock_time: 0.000021  Rows_sent: 2  Rows_examined: 2517002  Rows_affected: 0
# Bytes_sent: 975
SET timestamp=1542851901;
SELECT * FROM `x`  WHERE `xxx`