centos7搭建logstash

前两节已经成功完成ek的搭建，还剩最后的一个日志上传的功能

依次执行如下命令

cd /home/elk
wget https://artifacts.elastic.co/downloads/logstash/logstash-6.2.4.tar.gz
tar zxvf logstash-6.2..tar.gz
mv logstash-6.2. logstash

创建配置文件

cd logstash
cd data
vi logstash-simple.conf
监听8082端口

input {
stdin { }
tcp {
port => 8082
codec => json_lines
}
}
output {
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "myfistlog"
}
stdout { codec => rubydebug }
}

如果搭配filebeat使用，配置如下

input {
beats {
port => 8082
}
}
filter {

if [fields][logtype] == "claimzuul" {
json {
source => "message"
target => "data"
}
}
}
output {
if [fields][logtype] == "claimzuul"{
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "claimzuul-%{+YYYY.MM.dd}"
}
}
}

启动

[root@insure bin]# ./logstash -f /home/elk/logstash/data/logstash-simple.conf &
Sending Logstash's logs to /home/elk/logstash/logs which is now configured via log4j2.properties
[--27T15::,][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"netflow", :directory=>"/home/elk/logstash/modules/netflow/configuration"}
[--27T15::,][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/home/elk/logstash/modules/fb_apache/configuration"}
[--27T15::,][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.queue", :path=>"/home/elk/logstash/data/queue"}
[--27T15::,][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.dead_letter_queue", :path=>"/home/elk/logstash/data/dead_letter_queue"}
[--27T15::,][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[--27T15::,][INFO ][logstash.agent           ] No persistent UUID file found. Generating new UUID {:uuid=>"d4f319cf-97a7-4f90-83c6-8e62e8205fa5", :path=>"/home/elk/logstash/data/uuid"}
[--27T15::,][INFO ][logstash.runner          ] Starting Logstash {"logstash.version"=>"6.2.4"}
[--27T15::,][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>}
[--27T15::,][INFO ][logstash.pipeline        ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>, "pipeline.batch.size"=>, "pipeline.batch.delay"=>}
[--27T15::,][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://127.0.0.1:9200/]}}
[--27T15::,][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://127.0.0.1:9200/, :path=>"/"}
[--27T15::,][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>"http://127.0.0.1:9200/"}
[--27T15::,][INFO ][logstash.outputs.elasticsearch] ES Output version determined {:es_version=>}
[--27T15::,][WARN ][logstash.outputs.elasticsearch] Detected a .x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>6}
[--27T15::,][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil}
[--27T15::,][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
[--27T15::,][INFO ][logstash.outputs.elasticsearch] Installing elasticsearch template to _template/logstash
[--27T15::,][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//127.0.0.1:9200"]}
[--27T15::,][INFO ][logstash.pipeline        ] Pipeline started successfully {:pipeline_id=>"main", :thread=>"#<Thread:0x18db84e4 run>"}
The stdin plugin is now waiting for input:
[--27T15::,][INFO ][logstash.agent           ] Pipelines running {:count=>, :pipelines=>["main"]}