Logstash同步mysql数据库信息到ES

@font-face{ font-family:"Times New Roman"; } @font-face{ font-family:"宋体"; } @font-face{ font-family:"DejaVu Sans"; } @font-face{ font-family:"方正黑体_GBK"; } @font-face{ font-family:"Calibri"; } p.MsoNormal{ mso-style-name:正文; mso-style-parent:""; margin:0pt; margin-bottom:.0001pt; mso-pagination:none; text-align:justify; text-justify:inter-ideograph; font-family:Calibri; mso-fareast-font-family:宋体; mso-bidi-font-family:'Times New Roman'; font-size:10.5000pt; mso-font-kerning:1.0000pt; } h1{ mso-style-name:"标题 1"; mso-style-next:正文; margin-top:17.0000pt; margin-bottom:16.5000pt; mso-para-margin-top:0.0000gd; mso-para-margin-bottom:0.0000gd; page-break-after:avoid; mso-pagination:lines-together; text-align:justify; text-justify:inter-ideograph; mso-outline-level:1; line-height:240%; font-family:Calibri; mso-fareast-font-family:宋体; mso-bidi-font-family:'Times New Roman'; font-weight:bold; font-size:22.0000pt; mso-font-kerning:22.0000pt; } h2{ mso-style-name:"标题 2"; mso-style-noshow:yes; mso-style-next:正文; margin-top:13.0000pt; margin-bottom:13.0000pt; mso-para-margin-top:0.0000gd; mso-para-margin-bottom:0.0000gd; page-break-after:avoid; mso-pagination:lines-together; text-align:justify; text-justify:inter-ideograph; mso-outline-level:2; line-height:172%; font-family:'DejaVu Sans'; mso-fareast-font-family:方正黑体_GBK; mso-bidi-font-family:'Times New Roman'; font-weight:bold; font-size:16.0000pt; mso-font-kerning:1.0000pt; } span.msoIns{ mso-style-type:export-only; mso-style-name:""; text-decoration:underline; text-underline:single; color:blue; } span.msoDel{ mso-style-type:export-only; mso-style-name:""; text-decoration:line-through; color:red; } @page{mso-page-border-surround-header:no; mso-page-border-surround-footer:no;}@page Section0{ margin-top:72.0000pt; margin-bottom:72.0000pt; margin-left:90.0000pt; margin-right:90.0000pt; size:595.3000pt 841.9000pt; layout-grid:15.6000pt; } div.Section0{page:Section0;}

Logstash同步mysql数据库信息到ES

前提是ELK安装没问题

安装jdbc的数据连接插

使用logstash-plugin install logstash-input-jdbc 命令

进入到logstash安装目录下的bin目录（我的logstash安装目录：/usr/local/）

[root@es1 bin]# cd /usr/local/logstash-5.5.2/bin

[root@es1 bin]# ll

total 100

-rwxr-xr-x 1 root root   377 Aug 14  2017 cpdump

-rw-r--r-- 1 root root 15821 Dec 27 00:58 hs_err_pid1888.log

-rw-r--r-- 1 root root 15821 Dec 27 01:01 hs_err_pid1929.log

-rw-r--r-- 1 root root 15821 Dec 27 01:05 hs_err_pid2026.log

-rwxr-xr-x 1 root root   155 Aug 14  2017 ingest-convert.sh

-rwxr-xr-x 1 root root  1949 Aug 14  2017 logstash

-rw-r--r-- 1 root root   677 Aug 14  2017 logstash.bat

-rw-r--r-- 1 root root   756 Dec 27 16:30 logstash.conf

-rwxr-xr-x 1 root root  5400 Aug 14  2017 logstash.lib.sh

-rwxr-xr-x 1 root root   448 Aug 14  2017 logstash-plugin

-rw-r--r-- 1 root root   251 Aug 14  2017 logstash-plugin.bat

-rw-r--r-- 1 root root  1138 Dec 27 17:21 mysqltoes.conf

-rwxr-xr-x 1 root root   840 Aug 14  2017 ruby

-rw-r--r-- 1 root root  2795 Aug 14  2017 setup.bat

-rwxr-xr-x 1 root root  3530 Aug 14  2017 system-install

新建文件mysqltoes.conf

[root@es1 log]# vim /usr/local/logstash-5.5.2/bin/mysqltoes.conf

#

input {

stdin { }

jdbc {

#需要同步的数据库

jdbc_connection_string => "jdbc:mysql://192.168.100.112:3306/zabbix"

jdbc_user => "root"

jdbc_password => "123456"

#本地jar包

jdbc_driver_library => "/opt/mysql-connector-java-5.1.47-bin.jar"

jdbc_driver_class => "com.mysql.jdbc.Driver"

jdbc_paging_enabled => "true"

jdbc_page_size => "50000"

#获取到记录的SQL查询语句

statement => "SELECT * FROM users"

#定时字段 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新

schedule => "* * * * *"

}

}

output {

stdout {

codec => json_lines

}

elasticsearch {

#ESIP地址与端口

hosts => ["192.168.100.101:9200","192.168.100.102:9200","192.168.100.103:9200"]

#ES索引名称（自己定义的）

index => "users"

#文档类型

document_type => "user"

#文档类型id，%{userid}意思是取查询出来的userid的值，并将其映射到es中_id字段中

document_id => "%{userid}"

}

}

运行同步

./logstash -f mysqltoes.conf

查看同步结果