Spring Cloud Sleuth通过Kafka将链路追踪日志输出到ELK

1、工程简介

elk-eureka-server作为其他三个项目的服务注册中心

elk-kafka-client调用elk-kafka-server，elk-kafka-server再调用elk-kafka-server1

2、新建springboot项目elk-eureka-server

pom.xml内容如下

 <?xml version="1.0" encoding="UTF-8"?>

 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

     <modelVersion>4.0.0</modelVersion>

     <groupId>com.carry.elk</groupId>

     <artifactId>elk-eureka-server</artifactId>

     <version>0.0.1-SNAPSHOT</version>

     <packaging>jar</packaging>

     <name>elk-eureka-server</name>

     <description>Demo project for Spring Boot</description>

     <parent>

         <groupId>org.springframework.boot</groupId>

         <artifactId>spring-boot-starter-parent</artifactId>

         <version>2.0.3.RELEASE</version>

         <relativePath/> <!-- lookup parent from repository -->

     </parent>

     <properties>

         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

         <java.version>1.8</java.version>

         <spring-cloud.version>Finchley.RELEASE</spring-cloud.version>

     </properties>

     <dependencies>

         <dependency>

             <groupId>org.springframework.cloud</groupId>

             <artifactId>spring-cloud-starter-netflix-eureka-server</artifactId>

         </dependency>

         <dependency>

             <groupId>org.springframework.boot</groupId>

             <artifactId>spring-boot-starter-test</artifactId>

             <scope>test</scope>

         </dependency>

     </dependencies>

     <dependencyManagement>

         <dependencies>

             <dependency>

                 <groupId>org.springframework.cloud</groupId>

                 <artifactId>spring-cloud-dependencies</artifactId>

                 <version>${spring-cloud.version}</version>

                 <type>pom</type>

                 <scope>import</scope>

             </dependency>

         </dependencies>

     </dependencyManagement>

     <build>

         <plugins>

             <plugin>

                 <groupId>org.springframework.boot</groupId>

                 <artifactId>spring-boot-maven-plugin</artifactId>

             </plugin>

         </plugins>

     </build>

 </project>

application.yml内容如下

 server:

   port: 8761

 eureka:

   instance:

     hostname: localhost

   client:

     registerWithEureka: false

     fetchRegistry: false

     serviceUrl:

       defaultZone: http://${eureka.instance.hostname}:${server.port}/eureka/

在启动类ElkEurekaServerApplication上加上注解@EnableEurekaServer

3、新建springboot项目elk-kafka-client

pom.xml内容如下

 <?xml version="1.0" encoding="UTF-8"?>

 <project xmlns="http://maven.apache.org/POM/4.0.0"

     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

     <modelVersion>4.0.0</modelVersion>

     <groupId>com.carry.elk</groupId>

     <artifactId>elk-kafka-client</artifactId>

     <version>0.0.1-SNAPSHOT</version>

     <packaging>jar</packaging>

     <name>elk-kafka-client</name>

     <description>Demo project for Spring Boot</description>

     <parent>

         <groupId>org.springframework.boot</groupId>

         <artifactId>spring-boot-starter-parent</artifactId>

         <version>2.0.3.RELEASE</version>

         <relativePath /> <!-- lookup parent from repository -->

     </parent>

     <properties>

         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

         <java.version>1.8</java.version>

         <spring-cloud.version>Finchley.RELEASE</spring-cloud.version>

     </properties>

     <dependencies>

         <dependency>

             <groupId>org.springframework.boot</groupId>

             <artifactId>spring-boot-starter-web</artifactId>

         </dependency>

         <dependency>

             <groupId>org.springframework.cloud</groupId>

             <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>

         </dependency>

         <dependency>

             <groupId>org.springframework.cloud</groupId>

             <artifactId>spring-cloud-starter-openfeign</artifactId>

         </dependency>

         <dependency>

             <groupId>org.springframework.cloud</groupId>

             <artifactId>spring-cloud-starter-sleuth</artifactId>

         </dependency>

         <dependency>

             <groupId>net.logstash.logback</groupId>

             <artifactId>logstash-logback-encoder</artifactId>

             <version>4.11</version>

         </dependency>

         <dependency>

             <groupId>com.github.danielwegener</groupId>

             <artifactId>logback-kafka-appender</artifactId>

             <version>0.1.0</version>

             <scope>runtime</scope>

         </dependency>

         <dependency>

             <groupId>org.springframework.boot</groupId>

             <artifactId>spring-boot-devtools</artifactId>

             <scope>runtime</scope>

         </dependency>

         <dependency>

             <groupId>org.springframework.boot</groupId>

             <artifactId>spring-boot-starter-test</artifactId>

             <scope>test</scope>

         </dependency>

     </dependencies>

     <dependencyManagement>

         <dependencies>

             <dependency>

                 <groupId>org.springframework.cloud</groupId>

                 <artifactId>spring-cloud-dependencies</artifactId>

                 <version>${spring-cloud.version}</version>

                 <type>pom</type>

                 <scope>import</scope>

             </dependency>

         </dependencies>

     </dependencyManagement>

     <build>

         <plugins>

             <plugin>

                 <groupId>org.springframework.boot</groupId>

                 <artifactId>spring-boot-maven-plugin</artifactId>

             </plugin>

         </plugins>

     </build>

 </project>

application.yml内容如下

 server:

   port: 8080

 spring:

   application:

     name: ELK-KAFKA-CLIENT

   devtools:

     restart:

       enabled: true

 eureka:

   client:

     serviceUrl:

       defaultZone: http://localhost:8761/eureka/

在resources目录下新增logback.xml文件，内容如下

 <?xml version="1.0" encoding="UTF-8"?>

 <configuration debug="false" scan="true"

     scanPeriod="1 seconds">

     <include

         resource="org/springframework/boot/logging/logback/base.xml" />

     <!-- <jmxConfigurator/> -->

     <contextName>logback</contextName>

     <property name="log.path" value="\logs\logback.log" />

     <property name="log.pattern"

         value="%d{yyyy-MM-dd HH:mm:ss.SSS} -%5p ${PID} --- traceId:[%X{mdc_trace_id}] [%15.15t] %-40.40logger{39} : %m%n" />

     <appender name="file"

         class="ch.qos.logback.core.rolling.RollingFileAppender">

         <file>${log.path}</file>

         <encoder>

             <pattern>${log.pattern}</pattern>

         </encoder>

         <rollingPolicy

             class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

             <fileNamePattern>info-%d{yyyy-MM-dd}-%i.log

             </fileNamePattern>

             <timeBasedFileNamingAndTriggeringPolicy

                 class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">

                 <maxFileSize>10MB</maxFileSize>

             </timeBasedFileNamingAndTriggeringPolicy>

             <maxHistory>10</maxHistory>

         </rollingPolicy>

     </appender>

     <appender name="KafkaAppender"

         class="com.github.danielwegener.logback.kafka.KafkaAppender">

         <encoder

             class="com.github.danielwegener.logback.kafka.encoding.LayoutKafkaMessageEncoder">

             <layout class="net.logstash.logback.layout.LogstashLayout">

                 <includeContext>true</includeContext>

                 <includeCallerData>true</includeCallerData>

                 <customFields>{"system":"kafka"}</customFields>

                 <fieldNames

                     class="net.logstash.logback.fieldnames.ShortenedFieldNames" />

             </layout>

             <charset>UTF-8</charset>

         </encoder>

         <!--kafka topic 需要与配置文件里面的topic一致 否则kafka会沉默并鄙视你 -->

         <topic>applog</topic>

         <keyingStrategy

             class="com.github.danielwegener.logback.kafka.keying.HostNameKeyingStrategy" />

         <deliveryStrategy

             class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy" />

         <producerConfig>bootstrap.servers=192.168.68.110:9092,192.168.68.111:9092,192.168.68.112:9092</producerConfig>

     </appender>

     <!-- <logger name="Application_ERROR">

         <appender-ref ref="KafkaAppender" />

     </logger> -->

     <root level="info">

         <appender-ref ref="KafkaAppender" />

     </root> 

 </configuration>

新增Feign接口CilentFeignApi，内容如下

 package com.carry.elk;

 import org.springframework.cloud.openfeign.FeignClient;

 import org.springframework.http.MediaType;

 import org.springframework.web.bind.annotation.RequestMapping;

 import org.springframework.web.bind.annotation.RequestMethod;

 @FeignClient("ELK-KAFKA-SERVER")

 public interface CilentFeignApi {

     @RequestMapping(value = "/server", method = RequestMethod.GET, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)

     public String getString();

 }

新增ClientController，内容如下

 package com.carry.elk;

 import org.slf4j.Logger;

 import org.slf4j.LoggerFactory;

 import org.springframework.beans.factory.annotation.Autowired;

 import org.springframework.web.bind.annotation.GetMapping;

 import org.springframework.web.bind.annotation.RestController;

 @RestController

 public class ClientController {

 //    private final static Logger logger = LoggerFactory.getLogger("Application_ERROR");

     private final Logger logger = LoggerFactory.getLogger(this.getClass());

     @Autowired

     CilentFeignApi api;

     @GetMapping("/client")

     public String getString() {

         logger.info("开始调用服务端Server");

         return api.getString();

     }

 }

在启动类ElkKafkaClientApplication上加上注解@EnableEurekaClient和@EnableFeignClients

4、新建springboot项目elk-kafka-server

此项目跟elk-kafka-client项目相似只需作一下修改

application.yml文件中修改端口号为8081，application.name为ELK-KAFKA-SERVER

Feign接口中注解修改为@FeignClient("ELK-KAFKA-SERVER1")

Controller内容修改为

 package com.carry.elk;

 import org.slf4j.Logger;

 import org.slf4j.LoggerFactory;

 import org.springframework.beans.factory.annotation.Autowired;

 import org.springframework.web.bind.annotation.GetMapping;

 import org.springframework.web.bind.annotation.RestController;

 @RestController

 public class ServerController {

 //    private final static Logger logger = LoggerFactory.getLogger("Application_ERROR");

     private final Logger logger = LoggerFactory.getLogger(this.getClass());

     @Autowired

     ServerFeignApi api;

     @GetMapping("/server")

     public String getString() {

         logger.info("接收客户端调用并调用服务端Server1");

         return api.getString();

     }

 }

5、新建springboot项目elk-kafka-server1

此项目不需要调用其他项目所以去掉Feign的支持

application.yml文件中修改端口号为8082，application.name为ELK-KAFKA-SERVER1

Controller内容修改为

 package com.carry.elk;

 import org.slf4j.Logger;

 import org.slf4j.LoggerFactory;

 import org.springframework.web.bind.annotation.GetMapping;

 import org.springframework.web.bind.annotation.RestController;

 @RestController

 public class Server1Controller {

 //    private final static Logger logger = LoggerFactory.getLogger("Application_ERROR");

     private final Logger logger = LoggerFactory.getLogger(this.getClass());

     @GetMapping("/server")

     public String getString() {

         logger.info("接收服务端server的调用");

         return "I am server1.";

     }

 }

项目已写完让我们来看看结果吧

首先启动elk-eureka-server，然后依次启动elk-kafka-server1，elk-kafka-server，elk-kafka-client，浏览器中访问localhost:8761/，查看三个项目是否成功注册到eureka上去

访问localhost:8080/client，查看项目后台日志

这时我们可以去Kibana上查询日志了，访问http://192.168.68.112:5601/，进入Management创建index pattern kafka-logs-*，返回Discover页面