springboot配置kafka生产者和消费者详解

在原有pom.xml依赖下新添加一下kafka依赖ar包

<!--kafka-->

<dependency>

<groupId>org.springframework.kafka</groupId>

<artifactId>spring-kafka</artifactId>

<version>1.1.1.RELEASE</version>

</dependency>

<dependency>

<groupId>org.apache.kafka</groupId>

<artifactId>kafka_2.10</artifactId>

<version>0.10.0.1</version>

</dependency>

application.properties：

 #原始数据kafka读取

 kafka.consumer.servers=IP:9092,IP:9092(kafka消费集群ip+port端口)

 kafka.consumer.enable.auto.commit=true(是否自动提交)

 kafka.consumer.session.timeout=20000(连接超时时间)

 kafka.consumer.auto.commit.interval=100

 kafka.consumer.auto.offset.reset=latest(实时生产，实时消费，不会从头开始消费)

 kafka.consumer.topic=result(消费的topic)

 kafka.consumer.group.id=test(消费组)

 kafka.consumer.concurrency=10(设置消费线程数)

 #协议转换后存储kafka

 kafka.producer.servers=IP:9092,IP:9092(kafka生产集群ip+port端口)

 kafka.producer.topic=result(生产的topic)

 kafka.producer.retries=0

 kafka.producer.batch.size=4096

 kafka.producer.linger=1

 kafka.producer.buffer.memory=40960

springboot生产者配置：

 package com.mapbar.track_storage.config;

 import org.apache.kafka.clients.producer.ProducerConfig;

 import org.apache.kafka.common.serialization.StringSerializer;

 import org.springframework.beans.factory.annotation.Value;

 import org.springframework.context.annotation.Bean;

 import org.springframework.context.annotation.Configuration;

 import org.springframework.kafka.annotation.EnableKafka;

 import org.springframework.kafka.core.DefaultKafkaProducerFactory;

 import org.springframework.kafka.core.KafkaTemplate;

 import org.springframework.kafka.core.ProducerFactory;

 import java.util.HashMap;

 import java.util.Map;

 /**

 * kafka生产配置

 * @author Lvjiapeng

 *

 */

 @Configuration

 @EnableKafka

 public class KafkaProducerConfig {

 @Value("${kafka.producer.servers}")

 private String servers;

 @Value("${kafka.producer.retries}")

 private int retries;

 @Value("${kafka.producer.batch.size}")

 private int batchSize;

 @Value("${kafka.producer.linger}")

 private int linger;

 @Value("${kafka.producer.buffer.memory}")

 private int bufferMemory;

 public Map<String, Object> producerConfigs() {

 Map<String, Object> props = new HashMap<>();

 props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);

 props.put(ProducerConfig.RETRIES_CONFIG, retries);

 props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize);

 props.put(ProducerConfig.LINGER_MS_CONFIG, linger);

 props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory);

 props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);

 props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);

 return props;

 }

 public ProducerFactory<String, String> producerFactory() {

 return new DefaultKafkaProducerFactory<>(producerConfigs());

 }

 @Bean

 public KafkaTemplate<String, String> kafkaTemplate() {

 return new KafkaTemplate<String, String>(producerFactory());

 }

 }

springboot消费者配置：

 package com.mapbar.track_storage.config;

 import org.apache.kafka.clients.consumer.ConsumerConfig;

 import org.apache.kafka.common.serialization.StringDeserializer;

 import org.springframework.beans.factory.annotation.Value;

 import org.springframework.context.annotation.Bean;

 import org.springframework.context.annotation.Configuration;

 import org.springframework.kafka.annotation.EnableKafka;

 import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;

 import org.springframework.kafka.config.KafkaListenerContainerFactory;

 import org.springframework.kafka.core.ConsumerFactory;

 import org.springframework.kafka.core.DefaultKafkaConsumerFactory;

 import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;

 import java.util.HashMap;

 import java.util.Map;

 /**

 * kafka消费者配置

 * @author Lvjiapeng

 *

 */

 @Configuration

 @EnableKafka

 public class KafkaConsumerConfig {

 @Value("${kafka.consumer.servers}")

 private String servers;

 @Value("${kafka.consumer.enable.auto.commit}")

 private boolean enableAutoCommit;

 @Value("${kafka.consumer.session.timeout}")

 private String sessionTimeout;

 @Value("${kafka.consumer.auto.commit.interval}")

 private String autoCommitInterval;

 @Value("${kafka.consumer.group.id}")

 private String groupId;

 @Value("${kafka.consumer.auto.offset.reset}")

 private String autoOffsetReset;

 @Value("${kafka.consumer.concurrency}")

 private int concurrency;

 @Bean

 public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {

 ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();

 factory.setConsumerFactory(consumerFactory());

 factory.setConcurrency(concurrency);

 factory.getContainerProperties().setPollTimeout(1500);

 return factory;

 }

 public ConsumerFactory<String, String> consumerFactory() {

 return new DefaultKafkaConsumerFactory<>(consumerConfigs());

 }

 public Map<String, Object> consumerConfigs() {

 Map<String, Object> propsMap = new HashMap<>();

 propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);

 propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit);

 propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitInterval);

 propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeout);

 propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

 propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

 propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);

 propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset);

 return propsMap;

 }

 /**

 * kafka监听

 * @return

 */

 @Bean

 public RawDataListener listener() {

 return new RawDataListener();

 }

 }

生产者测试：

 package com.mapbar.track_storage.controller;

 import org.springframework.beans.factory.annotation.Autowired;

 import org.springframework.kafka.core.KafkaTemplate;

 import org.springframework.stereotype.Controller;

 import org.springframework.web.bind.annotation.RequestMapping;

 import org.springframework.web.bind.annotation.RequestMethod;

 import javax.servlet.http.HttpServletRequest;

 import javax.servlet.http.HttpServletResponse;

 import java.io.IOException;

 @RequestMapping(value = "/kafka")

 @Controller

 public class ProducerController {

 @Autowired

 private KafkaTemplate kafkaTemplate;

 @RequestMapping(value = "/producer",method = RequestMethod.GET)

 public void consume(HttpServletRequest request, HttpServletResponse response) throws IOException{

 String value = "{\"code\":200,\"dataVersion\":\"17q1\",\"message\":\"\",\"id\":\"364f79f28eea48eefeca8c85477a10d3\",\"source\":\"didi\",\"tripList\":[{\"subTripList\":[{\"startTimeStamp\":1519879598,\"schemeList\":[{\"distance\":0.0,\"ids\":\"94666702,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519879598,\"subTripId\":0},{\"startTimeStamp\":1519879727,\"schemeList\":[{\"distance\":1395.0,\"ids\":\"94666729,7298838,7291709,7291706,88613298,88613297,7297542,7297541,94698785,94698786,94698778,94698780,94698779,94698782,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519879812,\"subTripId\":1},{\"startTimeStamp\":1519879836,\"schemeList\":[{\"distance\":0.0,\"ids\":\"54123007,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519879904,\"subTripId\":2},{\"startTimeStamp\":1519879959,\"schemeList\":[{\"distance\":0.0,\"ids\":\"54190443,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519879959,\"subTripId\":3},{\"startTimeStamp\":1519880088,\"schemeList\":[{\"distance\":2885.0,\"ids\":\"94698824,94698822,94698789,94698786,54123011,54123012,54123002,94698763,94698727,94698722,94698765,54123006,54123004,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519880300,\"subTripId\":4},{\"startTimeStamp\":1519880393,\"schemeList\":[{\"distance\":2398.0,\"ids\":\"7309441,7303680,54123061,54123038,7309478,7309477,94698204,94698203,94698273,94698274,94698288,94698296,94698295,94698289,94698310,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519880636,\"subTripId\":5},{\"startTimeStamp\":1519881064,\"schemeList\":[{\"distance\":35.0,\"ids\":\"7309474,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519881204,\"subTripId\":6},{\"startTimeStamp\":1519881204,\"schemeList\":[{\"distance\":28.0,\"ids\":\"7309476,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519881266,\"subTripId\":7},{\"startTimeStamp\":1519881291,\"schemeList\":[{\"distance\":463.0,\"ids\":\"7303683,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519881329,\"subTripId\":8}],\"startTimeStamp\":1519879350,\"unUseTime\":1201,\"totalTime\":2049,\"endTimeStamp\":1519881399,\"tripId\":0}]}";

 for (int i = 1; i<=500; i++){

 kafkaTemplate.send("result",value);

 }

 }

 }

消费者测试：

 import net.sf.json.JSONObject;

 import org.apache.kafka.clients.consumer.ConsumerRecord;

 import org.apache.log4j.Logger;

 import org.springframework.beans.factory.annotation.Autowired;

 import org.springframework.kafka.annotation.KafkaListener;

 import org.springframework.stereotype.Component;

 import java.io.IOException;

 import java.util.List;

 /**

 * kafka监听

 * @author shangzz

 *

 */

 @Component

 public class RawDataListener {

 Logger logger=Logger.getLogger(RawDataListener.class);

 @Autowired

 private MatchRoadService matchRoadService;

 /**

 * 实时获取kafka数据(生产一条，监听生产topic自动消费一条)

 * @param record

 * @throws IOException

 */

 @KafkaListener(topics = {"${kafka.consumer.topic}"})

 public void listen(ConsumerRecord<?, ?> record) throws IOException {

 String value = (String) record.value();

 System.out.println(value);

 }

 }

总结：

① 生产者环境类配置好以后，@Autowired自动注入KafkaTemplate类，使用send方法生产消息

② 消费者环境类配置好以后，方法头前使用@KafkaListener(topics = {"${kafka.consumer.topic}"})注解监听topic并传入ConsumerRecord<?, ?> record对象即可自动消费topic

③ 相关kafka配置只需在application.properties照葫芦画瓢添加，修改或者删除配置并在环境配置类中做出相应修改即可

springboot配置kafka生产者和消费者详解的更多相关文章

kafka学习（三）kafka生产者，消费者详解
文章更新时间:2020/06/14 一.生产者当我们发送消息之前,先问几个问题:每条消息都是很关键且不能容忍丢失么?偶尔重复消息可以么?我们关注的是消息延迟还是写入消息的吞吐量? 举个例子,有一个信 ...
kafka生产者和消费者流程
前言根据源码分析kafka java客户端的生产者和消费者的流程. 基于zookeeper的旧消费者 kafka消费者从消费数据到关闭经历的流程. 由于3个核心线程基于zookeeper的连接器监 ...
SpringBoot与PageHelper的整合示例详解
SpringBoot与PageHelper的整合示例详解 1.PageHelper简介 PageHelper官网地址: https://pagehelper.github.io/ 摘要: com.gi ...
kafka生产者和消费者api的简单使用
kafka生产者和消费者api的简单使用一.背景二.需要实现的功能 1.生产者实现功能 1.KafkaProducer线程安全的,可以在多线程中使用. 2.消息发送的key和value的序列化 3 ...
GPIO 配置之ODR, BSRR, BRR 详解
STM32 GPIO 配置之ODR, BSRR, BRR 详解用stm32 的配置GPIO 来控制LED 显示状态,可用ODR,BSRR,BRR 直接来控制引脚输出状态. ODR寄存器可读可写:既能 ...
[Spark][kafka]kafka 生产者，消费者互动例子
[Spark][kafka]kafka 生产者,消费者互动例子 # pwd/usr/local/kafka_2.11-0.10.0.1/bin 创建topic:# ./kafka-topics.sh ...
centos7.2环境nginx+mysql+php-fpm+svn配置walle自动化部署系统详解
centos7.2环境nginx+mysql+php-fpm+svn配置walle自动化部署系统详解操作系统:centos 7.2 x86_64 安装walle系统服务端 1.以下安装,均在宿主机( ...
STM32 GPIO 配置之ODR, BSRR, BRR 详解
STM32 GPIO 配置之ODR, BSRR, BRR 详解用stm32 的配置GPIO 来控制LED 显示状态,可用ODR,BSRR,BRR 直接来控制引脚输出状态. ODR寄存器可读可写:既能 ...
Kafka 学习之路（四）—— Kafka消费者详解
一.消费者和消费者群组在Kafka中,消费者通常是消费者群组的一部分,多个消费者群组共同读取同一个主题时,彼此之间互不影响.Kafka之所以要引入消费者群组这个概念是因为Kafka消费者经常会做一些 ...

随机推荐

在notepad++中快速插入当前时间方法
插件是notepad++的一大优势,而要实现此功能,也必须借助TextFX插件. 1.点击"插件"-->"Plugin Manager"-->&qu ...
Windows Socket和Linux Socket编程的区别 ZZ
socket相关程序从Windows移植到Linux下需要注意的: 1)头文件 Windows下winsock.h/winsock2.h Linux下sys/socket.h 错误处理:errno.h ...
Hadoop配置文件详解
1 获取默认配置配置hadoop,主要是配置core-site.xml,hdfs-site.xml,mapred-site.xml三个配置文件,默认下来,这些配置文件都是空的,所以很难知 ...
ASProgressPopUpView
ASProgressPopUpView https://github.com/alskipp/ASProgressPopUpView 效果: -使用- 将源码拖入工程当中: // // RootVie ...
C/S架构的性能测试
很多人关心LR在C/S架构上如何实施性能测试,我想根本原因在于两个方面,一是很多时候脚本无法录制,即LR无法成功调用被测的应用程序,二是测试脚本即使录制下来,可读性不强,往往不能运行通过,调试时无从下 ...
利用describe( )中的count来检查数据是否缺省
#-*- coding: utf-8 -*- #在python的pandas库中,只需要读入数据,然后使用describe()函数就可以查看数据的基本情况 import pandas as pd in ...
四种Timer的区别和用法
1.System.Threading.Timer 线程计时器 1.最底层.轻量级的计时器.基于线程池实现的,工作在辅助线程. 2.它并不是内在线程安全的,并且使用起来比其他计时器更麻烦.此计时器通常不 ...
API接口数据自检
这个周末的娱乐,通用模块,让后端自检,严格客户端按照文档的要求来,妈妈再也不担心我加班了,对某些团队来说,可能根本用不着,本是想到就尝试一把而已. 哎,傻X的客户端程序员,时间都去推辞扯淡打扮啦,好好 ...
0-创建scott数据
CREATE TABLE dept ( deptno INT PRIMARY KEY, dname VARCHAR(14), loc VARCHAR(13) ); INSERT INTO d ...
python中的BaseManager通信（二）文件二分
提供服务部分(运行时在接收端未打开前不能关闭) #mainsec.py from multiprocessing import Process, Queue from multiprocessing. ...

springboot配置kafka生产者和消费者详解

springboot配置kafka生产者和消费者详解的更多相关文章

随机推荐

热门专题