Java读文件写入kafka

目录

• Java读文件写入kafka
  • 文件格式
  • pom依赖
  • java代码

Java读文件写入kafka

文件格式

840271		103208		0	0.0	insert	84e66588-8875-4411-9cc6-0ac8302408bf	3	2	4	wangxiao	0.0	0	0.0	9927525	1619330049000	normal	1bd221d7380546be9fe8e10a63cf8130	0	0	NULL	0	0	Qw==	4253976	79
840271		103208		0	0.0	insert	cece91f8-8a17-4417-84d8-f6293849e187	3	2	4	wangxiao	0.0	0	0.0	9927525	1619330049000	normal	38204d736e8646fd956131409fc4196e	0	0	NULL	0	0	Qw==	4002760	80

pom依赖

 <dependencies>
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka-clients</artifactId>
      <version>0.11.0.0</version>
      <scope>provided</scope>
    </dependency>
</dependencies>

<build>
        <!--编译的文件目录-->
        <sourceDirectory>src/main/scala</sourceDirectory>
        <resources>
            <resource>
                <directory>src/main/resources</directory>
            </resource>
        </resources>
        <plugins>
            <!-- build-helper-maven-plugin, 设置多个源文件夹 -->
            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>build-helper-maven-plugin</artifactId>
                <version>3.0.0</version>
                <executions>
                    <execution>
                        <id>add-source</id>
                        <phase>generate-sources</phase>
                        <goals>
                            <goal>add-source</goal>
                        </goals>
                        <configuration>
                            <sources>
                                <source>src/main/java</source>
                                <source>src/main/scala</source>
                                <!-- 我们可以通过在这里添加多个source节点，来添加任意多个源文件夹 -->
                            </sources>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                    <archive>
                        <manifest>
                            <mainClass>com.xueersi.bdc.flink.WordCount</mainClass>
                        </manifest>
                    </archive>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <!-- Java Compiler -->
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.1</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
            <!--Scala Compiler-->
            <plugin>
                <groupId>net.alchim31.maven</groupId>
                <artifactId>scala-maven-plugin</artifactId>
                <version>3.2.2</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

java代码

import com.alibaba.fastjson.JSON;
import com.bdc.flink.slove_problem.Ans5;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Properties;

/**
 * @description: 读取D2数据（场景5）写入kafka
 * @author: HaoWu
 * @create: 2021年04月26日
 */
public class D2ToKafka {
    public static void main(String[] args) throws IOException, InterruptedException {
//        String bootstrap_servers = "10.90.XXXX:9092,10.90.XXXX:9092,10.90.XXXX:9092"; // 输出kafak路径
//        String topic = "test20585696test"; //输出topic
//        String path = "/Users/haowu/software/d2_test";

        String bootstrap_servers= args[0]; // 输出kafak路径
        String topic=args[1]; //输出topic
        String path = args[2]; //输入文件路径

        Properties props = new Properties();
        props.put("bootstrap.servers", bootstrap_servers);//maxwell 测试kafka集群
        props.put("acks", "all");
        props.put("retries", 1);//重试次数
        props.put("batch.size", 16384);//批次大小
        props.put("linger.ms", 1);//等待时间
        props.put("buffer.memory", 33554432);//RecordAccumulator缓冲区大小
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        Producer<String, String> producer = new KafkaProducer<>(props);

        readTxt2Json(path, producer, topic);

    }

    public static void readTxt2Json(String path, Producer producer, String topic) throws IOException, InterruptedException {
        File file = new File(path);
        FileInputStream fis = null;
        InputStreamReader isr = null;
        BufferedReader br = null;
        try {
            fis = new FileInputStream(file);
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        }
        try {
            isr = new InputStreamReader(fis, "utf-8");
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }
        br = new BufferedReader(isr);
        String line = null;
        System.out.println("================== start ===================：" + System.currentTimeMillis());
        while ((line = br.readLine()) != null) {

            Ans5 ans5 = str2JsonStr(line);
            String key = ans5.getStu_id();
            String value = JSON.toJSONString(ans5);
            System.out.println(value);

            // 写入kafka
            producer.send(new ProducerRecord<>(topic, key, value));
        }
        //System.out.println(jsonStr);

        //关闭produce
        producer.close();
        System.out.println("================== end ===================：" + System.currentTimeMillis());

    }

    /**
     * 构建场景5作答bean，字符串转json字符
     *
     * @param str
     * @return
     */
    public static Ans5 str2JsonStr(String str) {
        String[] datas = str.split("\t");
        D2D3Bean bean = new D2D3Bean(datas[0], datas[1], datas[2], datas[3], datas[4], datas[5]
                , datas[6], datas[7], datas[8], datas[9], datas[10]
                , datas[11], datas[12], datas[13], datas[14], datas[15]
                , datas[16], datas[17], datas[18], datas[19], datas[20], datas[21], datas[22]
                , datas[23], datas[24], datas[25], datas[26]);

        return new Ans5(bean.getStu_id(), bean.getCourse_id(), bean.getPlan_id(), bean.getQues_id(), bean.getUser_answer(), bean.getAnswer_duration(),
               fromTimestampToHour(bean.getSub_time()), bean.getAnswer_status(), bean.getUuid(), bean.getOperate_type(), bean.getAns_scene(), bean.getRecom_id(), bean.getGrade_id(),
                bean.getSubject_id(), bean.getOrg_code(), bean.getQue_score(), bean.getStu_score(), bean.getScene_code(), bean.getQue_sort(), bean.getTest_category(), bean.getExam_id(), bean.getTest_num()
        );
    }

    /**
     * 毫秒时间戳->yyyy-MM-dd HH:mm:ss
     * @param ts
     * @return
     */
    public static String fromTimestampToHour(String ts){
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        Date date = new Date(Long.valueOf(ts));
        return  simpleDateFormat.format(date);
    }

}