docker部署logstash

1、下载镜像

[root@vanje-dev01 ~]# docker pull logstash:7.0.1

2、安装部署

2.1  创建宿主映射目录

# mkdir /etc/logstash/

# mkdir /etc/logstash/conf.d

2.2 编辑配置文件

# vim /etc/logstash/logstash.yml

http.host: "0.0.0.0"

# touch /etc/logstash/patterns

# chown -R 1000:1000 /etc/logstash/

2.3 启动

# docker run -d --name logstash \

-p 5044:5044 -p 9600:9600 \

--net=host --restart=always \

-v /home/logstash:/usr/share/logstash \

logstash:7.1.1

目录映射说明：

　　1、/etc/logstash/logstash.yml：logstash启动配置文件

　　2、/etc/logstash/conf.d： logstash过滤日志规则

　　3、/etc/logstash/patterns： 自定义匹配规则

生产中为了保证elk集群的稳定性，Logstash作为日志的解析分割，一般都会在前面添加redis或者kafka集群。

springcloud+logback+logstash+rabbitmq

pom

<!--logback日志写入rabbitmq-->

<dependency>

<groupId>org.springframework.amqp</groupId>

<artifactId>spring-rabbit</artifactId>

</dependency>

<dependency>

<groupId>org.springframework.boot</groupId>

<artifactId>spring-boot-starter-amqp</artifactId>

</dependency>

logback-spring.xml配置

<?xml version="1.0" encoding="UTF-8"?>

<configuration debug="false">

<!--定义日志文件的存储地址 -->

<property name="LOG_HOME" value="../logs/system" />

<!--<property name="COLOR_PATTERN" value="%black(%contextName-) %red(%d{yyyy-MM-dd HH:mm:ss}) %green([%thread]) %highlight(%-5level) %boldMagenta( %replace(%caller{1}){'\t|Caller.{1}0|\r\n', ''})- %gray(%msg%xEx%n)" />-->

<!-- 控制台输出 -->

<!--<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">-->

<!-- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">-->

<!-- <!–格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符-->

<!-- <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}:%L - %msg%n</pattern>–>-->

<!-- <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{50}:%L) - %msg%n</pattern>-->

<!-- </encoder>-->

<!--</appender>-->

<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">

<layout class="ch.qos.logback.classic.PatternLayout">

<pattern>%highlight(%d{yyyy-MM-dd HH:mm:ss.SSS}) %boldYellow([%thread]) %-5level %logger{36}- %msg%n</pattern>

</layout>

</appender>

<!-- 按照每天生成日志文件 -->

<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">

<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

<!--日志文件输出的文件名 -->

<FileNamePattern>${LOG_HOME}/jeecgboot-system-%d{yyyy-MM-dd}.log</FileNamePattern>

<!--日志文件保留天数 -->

<MaxHistory>7</MaxHistory>

</rollingPolicy>

<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

<!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符 -->

<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}:%L - %msg%n</pattern>

</encoder>

</appender>

<!-- 生成 error html格式日志开始 -->

<appender name="ERROR_LOG" class="ch.qos.logback.core.rolling.RollingFileAppender">

<filter class="ch.qos.logback.classic.filter.ThresholdFilter">

<!--设置日志级别,过滤掉info日志,只输入error日志-->

<level>ERROR</level>

</filter>

<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

<!--<encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">-->

<!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符 -->

<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}:%L - %msg%n</pattern>

<!--<layout class="ch.qos.logback.classic.html.HTMLLayout">-->

<!-- <pattern>%p%d%msg%M%F{32}%L</pattern>-->

<!--</layout>-->

</encoder>

<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

<!--日志文件输出的文件名 -->

<FileNamePattern>${LOG_HOME}/error-log-%d{yyyy-MM-dd}.log</FileNamePattern>

<!--<!–日志文件保留天数 –>-->

<MaxHistory>7</MaxHistory>

</rollingPolicy>

<!--<file>${LOG_HOME}/error-log.html</file>-->

</appender>

<!--logback日志写入rabbitmq -->

<appender name="RabbitMq"

class="org.springframework.amqp.rabbit.logback.AmqpAppender">

<layout>

<pattern><![CDATA[ %d %p %t [%c] - <%m>%n ]]></pattern>

</layout>

<!--rabbitmq地址 -->

<addresses>192.168.0.20:5670</addresses>

<abbreviation>36</abbreviation>

<includeCallerData>true</includeCallerData>

<applicationId>system</applicationId>

<username>admin</username>

<password>admin</password>

<!--路邮件 -->

<!--{applicationId}  -->

<!--%property{applicationId}.%c.%p 收集不到日志  -->

<!--最后改为精确匹配了  非常重要 -->

<routingKeyPattern>system</routingKeyPattern>

<generateId>true</generateId>

<charset>UTF-8</charset>

<durable>true</durable>

<deliveryMode>NON_PERSISTENT</deliveryMode>

<declareExchange>true</declareExchange>

<autoDelete>false</autoDelete>

</appender>

<!--myibatis log configure -->

<logger name="com.apache.ibatis" level="TRACE" />

<logger name="java.sql.Connection" level="DEBUG" />

<logger name="java.sql.Statement" level="DEBUG" />

<logger name="java.sql.PreparedStatement" level="DEBUG" />

<!-- 日志输出级别 -->

<root level="INFO">

<appender-ref ref="STDOUT" />

<appender-ref ref="FILE" />

<appender-ref ref="ERROR_LOG" />

<!--这块也要添加-->

<appender-ref ref="RabbitMq" />

</root>

</configuration>

Logstash 配置

vi /home/logstash/config/conf.d/system-log.conf

input {

rabbitmq {

type =>"all"

durable => true

exchange => "logs"

exchange_type => "topic"

key => "system"

host => "192.168.0.20:5670"

#          port => 5670

user => "admin"

password => "admin"

queue => "system-mq"

auto_delete => false

}

}

output {

elasticsearch {

hosts => ["192.168.0.23:9200"]

index => "system_log_mq"

}

}