springboot配置kafka生产者和消费者详解

在原有pom.xml依赖下新添加一下kafka依赖ar包

<!--kafka-->
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>1.1.1.RELEASE</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>0.10.0.1</version>
</dependency>

application.properties：

 #原始数据kafka读取
 kafka.consumer.servers=IP:9092,IP:9092(kafka消费集群ip+port端口)
 kafka.consumer.enable.auto.commit=true(是否自动提交)
 kafka.consumer.session.timeout=20000(连接超时时间)
 kafka.consumer.auto.commit.interval=100
 kafka.consumer.auto.offset.reset=latest(实时生产，实时消费，不会从头开始消费)
 kafka.consumer.topic=result(消费的topic)
 kafka.consumer.group.id=test(消费组)
 kafka.consumer.concurrency=10(设置消费线程数)

 #协议转换后存储kafka
 kafka.producer.servers=IP:9092,IP:9092(kafka生产集群ip+port端口)
 kafka.producer.topic=result(生产的topic)
 kafka.producer.retries=0
 kafka.producer.batch.size=4096
 kafka.producer.linger=1
 kafka.producer.buffer.memory=40960

springboot生产者配置：

 package com.mapbar.track_storage.config;

 import org.apache.kafka.clients.producer.ProducerConfig;
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.kafka.annotation.EnableKafka;
 import org.springframework.kafka.core.DefaultKafkaProducerFactory;
 import org.springframework.kafka.core.KafkaTemplate;
 import org.springframework.kafka.core.ProducerFactory;

 import java.util.HashMap;
 import java.util.Map;

 /**
 * kafka生产配置
 * @author Lvjiapeng
 *
 */
 @Configuration
 @EnableKafka
 public class KafkaProducerConfig {
 @Value("${kafka.producer.servers}")
 private String servers;
 @Value("${kafka.producer.retries}")
 private int retries;
 @Value("${kafka.producer.batch.size}")
 private int batchSize;
 @Value("${kafka.producer.linger}")
 private int linger;
 @Value("${kafka.producer.buffer.memory}")
 private int bufferMemory;

 public Map<String, Object> producerConfigs() {
 Map<String, Object> props = new HashMap<>();
 props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
 props.put(ProducerConfig.RETRIES_CONFIG, retries);
 props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize);
 props.put(ProducerConfig.LINGER_MS_CONFIG, linger);
 props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory);
 props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
 props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
 return props;
 }

 public ProducerFactory<String, String> producerFactory() {
 return new DefaultKafkaProducerFactory<>(producerConfigs());
 }

 @Bean
 public KafkaTemplate<String, String> kafkaTemplate() {
 return new KafkaTemplate<String, String>(producerFactory());
 }
 }

springboot消费者配置：

 package com.mapbar.track_storage.config;

 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.common.serialization.StringDeserializer;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.kafka.annotation.EnableKafka;
 import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
 import org.springframework.kafka.config.KafkaListenerContainerFactory;
 import org.springframework.kafka.core.ConsumerFactory;
 import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
 import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;

 import java.util.HashMap;
 import java.util.Map;

 /**
 * kafka消费者配置
 * @author Lvjiapeng
 *
 */
 @Configuration
 @EnableKafka
 public class KafkaConsumerConfig {

 @Value("${kafka.consumer.servers}")
 private String servers;
 @Value("${kafka.consumer.enable.auto.commit}")
 private boolean enableAutoCommit;
 @Value("${kafka.consumer.session.timeout}")
 private String sessionTimeout;
 @Value("${kafka.consumer.auto.commit.interval}")
 private String autoCommitInterval;
 @Value("${kafka.consumer.group.id}")
 private String groupId;
 @Value("${kafka.consumer.auto.offset.reset}")
 private String autoOffsetReset;
 @Value("${kafka.consumer.concurrency}")
 private int concurrency;

 @Bean
 public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
 ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
 factory.setConsumerFactory(consumerFactory());
 factory.setConcurrency(concurrency);
 factory.getContainerProperties().setPollTimeout(1500);
 return factory;
 }

 public ConsumerFactory<String, String> consumerFactory() {
 return new DefaultKafkaConsumerFactory<>(consumerConfigs());
 }

 public Map<String, Object> consumerConfigs() {
 Map<String, Object> propsMap = new HashMap<>();
 propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
 propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit);
 propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitInterval);
 propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeout);
 propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
 propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
 propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
 propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset);
 return propsMap;
 }
 /**
 * kafka监听
 * @return
 */
 @Bean
 public RawDataListener listener() {
 return new RawDataListener();
 }

 }

生产者测试：

 package com.mapbar.track_storage.controller;

 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.kafka.core.KafkaTemplate;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;

 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;

 @RequestMapping(value = "/kafka")
 @Controller
 public class ProducerController {
 @Autowired
 private KafkaTemplate kafkaTemplate;

 @RequestMapping(value = "/producer",method = RequestMethod.GET)
 public void consume(HttpServletRequest request, HttpServletResponse response) throws IOException{
 String value = "{\"code\":200,\"dataVersion\":\"17q1\",\"message\":\"\",\"id\":\"364f79f28eea48eefeca8c85477a10d3\",\"source\":\"didi\",\"tripList\":[{\"subTripList\":[{\"startTimeStamp\":1519879598,\"schemeList\":[{\"distance\":0.0,\"ids\":\"94666702,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519879598,\"subTripId\":0},{\"startTimeStamp\":1519879727,\"schemeList\":[{\"distance\":1395.0,\"ids\":\"94666729,7298838,7291709,7291706,88613298,88613297,7297542,7297541,94698785,94698786,94698778,94698780,94698779,94698782,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519879812,\"subTripId\":1},{\"startTimeStamp\":1519879836,\"schemeList\":[{\"distance\":0.0,\"ids\":\"54123007,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519879904,\"subTripId\":2},{\"startTimeStamp\":1519879959,\"schemeList\":[{\"distance\":0.0,\"ids\":\"54190443,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519879959,\"subTripId\":3},{\"startTimeStamp\":1519880088,\"schemeList\":[{\"distance\":2885.0,\"ids\":\"94698824,94698822,94698789,94698786,54123011,54123012,54123002,94698763,94698727,94698722,94698765,54123006,54123004,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519880300,\"subTripId\":4},{\"startTimeStamp\":1519880393,\"schemeList\":[{\"distance\":2398.0,\"ids\":\"7309441,7303680,54123061,54123038,7309478,7309477,94698204,94698203,94698273,94698274,94698288,94698296,94698295,94698289,94698310,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519880636,\"subTripId\":5},{\"startTimeStamp\":1519881064,\"schemeList\":[{\"distance\":35.0,\"ids\":\"7309474,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519881204,\"subTripId\":6},{\"startTimeStamp\":1519881204,\"schemeList\":[{\"distance\":28.0,\"ids\":\"7309476,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519881266,\"subTripId\":7},{\"startTimeStamp\":1519881291,\"schemeList\":[{\"distance\":463.0,\"ids\":\"7303683,\",\"schemeId\":0,\"linkList\":[{\"score\":72,\"distance\":1,\"gpsList\":[{\"origLonLat\":\"116.321343,40.43242\",\"grabLonLat\":\"112.32312,40.32132\",\"timestamp\":1515149926000}]}]}],\"endTimeStamp\":1519881329,\"subTripId\":8}],\"startTimeStamp\":1519879350,\"unUseTime\":1201,\"totalTime\":2049,\"endTimeStamp\":1519881399,\"tripId\":0}]}";
 for (int i = 1; i<=500; i++){
 kafkaTemplate.send("result",value);
 }
 }
 }

消费者测试：

 import net.sf.json.JSONObject;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.log4j.Logger;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.kafka.annotation.KafkaListener;
 import org.springframework.stereotype.Component;

 import java.io.IOException;
 import java.util.List;

 /**
 * kafka监听
 * @author shangzz
 *
 */
 @Component
 public class RawDataListener {
 Logger logger=Logger.getLogger(RawDataListener.class);
 @Autowired
 private MatchRoadService matchRoadService;

 /**
 * 实时获取kafka数据(生产一条，监听生产topic自动消费一条)
 * @param record
 * @throws IOException
 */
 @KafkaListener(topics = {"${kafka.consumer.topic}"})
 public void listen(ConsumerRecord<?, ?> record) throws IOException {
 String value = (String) record.value();
 System.out.println(value);
 }

 }

总结：

①  生产者环境类配置好以后，@Autowired自动注入KafkaTemplate类，使用send方法生产消息

②  消费者环境类配置好以后，方法头前使用@KafkaListener(topics = {"${kafka.consumer.topic}"})注解监听topic并传入ConsumerRecord<?, ?> record对象即可自动消费topic

③  相关kafka配置只需在application.properties照葫芦画瓢添加，修改或者删除配置并在环境配置类中做出相应修改即可