Spark Streaming接收Kafka数据存储到Hbase

Spark Streaming接收Kafka数据存储到Hbase

fly

spark

hbase

kafka

主要参考了这篇文章https://yq.aliyun.com/articles/60712（[点我]）(https://yq.aliyun.com/articles/60712), 不过这篇文章使用的spark貌似是spark1.x的。我这里主要是改为了spark2.x的方式

kafka生产数据

闲话少叙，直接上代码：

1. import java.util.{Properties, UUID} 
   

   ---

2. ---

3. import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} 
   

   ---

4. import org.apache.kafka.common.serialization.StringSerializer 
   

   ---

5. ---

6. import scala.util.Random 
   

   ---

7. ---

8. ---

9. object KafkaProducerTest { 
   

   ---

10. def main(args: Array[String]): Unit = { 
    

    ---

11. val rnd = new Random() 
    

    ---

12. // val topics = "world" 
    

    ---

13. val topics = "test" 
    

    ---

14. val brokers = "localhost:9092" 
    

    ---

15. val props = new Properties() 
    

    ---

16. props.put("delete.topic.enable", "true") 
    

    ---

17. props.put("key.serializer", classOf[StringSerializer]) 
    

    ---

18. // props.put("value.serializer", "org.apache.kafka.common.serialization.StringDeserializer") 
    

    ---

19. props.put("value.serializer", classOf[StringSerializer]) 
    

    ---

20. props.put("bootstrap.servers", brokers) 
    

    ---

21. //props.put("batch.num.messages","10");//props.put("batch.num.messages","10"); 
    

    ---

22. ---

23. //props.put("queue.buffering.max.messages", "20"); 
    

    ---

24. //linger.ms should be 0~100 ms 
    

    ---

25. props.put("linger.ms", "50") 
    

    ---

26. //props.put("block.on.buffer.full", "true"); 
    

    ---

27. //props.put("max.block.ms", "100000"); 
    

    ---

28. //batch.size and buffer.memory should be changed with "the kafka message size and message sending speed" 
    

    ---

29. props.put("batch.size", "16384") 
    

    ---

30. props.put("buffer.memory", "1638400") 
    

    ---

31. ---

32. props.put("queue.buffering.max.messages", "1000000") 
    

    ---

33. props.put("queue.enqueue.timeout.ms", "20000000") 
    

    ---

34. props.put("producer.type", "sync") 
    

    ---

35. ---

36. val producer = new KafkaProducer[String,String](props) 
    

    ---

37. for(i <- 1001 to 2000){ 
    

    ---

38. val key = UUID.randomUUID().toString.substring(0,5) 
    

    ---

39. val value = "fly_" + i + "_" + key 
    

    ---

40. producer.send(new ProducerRecord[String, String](topics,key, value))//.get() 
    

    ---

41. ---

42. } 
    

    ---

43. ---

44. producer.flush() 
    

    ---

45. } 
    

    ---

46. } 
    

    ---

47. ---

生产的数据格式为（key,value) = （uuid, fly_i_key） 的形式

spark streaming 读取kafka并保存到hbase

当kafka里面有数据后，使用spark streaming 读取，并存。直接上代码：

1. import java.util.UUID 
   

   ---

2. ---

3. import org.apache.hadoop.hbase.HBaseConfiguration 
   

   ---

4. import org.apache.hadoop.hbase.client.{Mutation, Put} 
   

   ---

5. import org.apache.hadoop.hbase.io.ImmutableBytesWritable 
   

   ---

6. import org.apache.hadoop.hbase.mapreduce.TableOutputFormat 
   

   ---

7. import org.apache.hadoop.hbase.util.Bytes 
   

   ---

8. import org.apache.hadoop.mapred.JobConf 
   

   ---

9. import org.apache.hadoop.mapreduce.OutputFormat 
   

   ---

10. import org.apache.kafka.clients.consumer.ConsumerRecord 
    

    ---

11. import org.apache.kafka.common.serialization.StringDeserializer 
    

    ---

12. import org.apache.spark.rdd.RDD 
    

    ---

13. import org.apache.spark.sql.SparkSession 
    

    ---

14. import org.apache.spark.streaming.kafka010.ConsumerStrategies.Subscribe 
    

    ---

15. import org.apache.spark.streaming.kafka010.KafkaUtils 
    

    ---

16. import org.apache.spark.streaming.kafka010.LocationStrategies.PreferConsistent 
    

    ---

17. import org.apache.spark.streaming.{Seconds, StreamingContext} 
    

    ---

18. ---

19. /** 
    

    ---

20. * spark streaming 写入到hbase 
    

    ---

21. * Sparkstreaming读取Kafka消息再结合SparkSQL，将结果保存到HBase 
    

    ---

22. */ 
    

    ---

23. ---

24. ---

25. object OBDSQL { 
    

    ---

26. case class Person(name: String, age: Int, key: String) 
    

    ---

27. ---

28. def main(args: Array[String]): Unit = { 
    

    ---

29. val spark = SparkSession 
    

    ---

30. .builder() 
    

    ---

31. .appName("sparkSql") 
    

    ---

32. .master("local[4]") 
    

    ---

33. .getOrCreate() 
    

    ---

34. ---

35. val sc = spark.sparkContext 
    

    ---

36. ---

37. val ssc = new StreamingContext(sc, Seconds(5)) 
    

    ---

38. ---

39. val topics = Array("test") 
    

    ---

40. val kafkaParams = Map( 
    

    ---

41. "bootstrap.servers" -> "localhost:9092,anotherhost:9092", 
    

    ---

42. "key.deserializer" -> classOf[StringDeserializer], 
    

    ---

43. "value.deserializer" -> classOf[StringDeserializer], 
    

    ---

44. // "group.id" -> "use_a_separate_group_id_for_each_stream", 
    

    ---

45. "group.id" -> "use_a_separate_group_id_for_each_stream_fly", 
    

    ---

46. // "auto.offset.reset" -> "latest", 
    

    ---

47. "auto.offset.reset" -> "earliest", 
    

    ---

48. // "auto.offset.reset" -> "none", 
    

    ---

49. "enable.auto.commit" -> (false: java.lang.Boolean) 
    

    ---

50. ) 
    

    ---

51. ---

52. val lines = KafkaUtils.createDirectStream[String, String]( 
    

    ---

53. ssc, 
    

    ---

54. PreferConsistent, 
    

    ---

55. Subscribe[String, String](topics, kafkaParams) 
    

    ---

56. ) 
    

    ---

57. ---

58. // lines.map(record => (record.key, record.value)).print() 
    

    ---

59. // lines.map(record => record.value.split("_")).map(x=> (x(0),x(1), x(2))).print() 
    

    ---

60. ---

61. lines.foreachRDD((rdd: RDD[ConsumerRecord[String, String]]) => { 
    

    ---

62. import spark.implicits._ 
    

    ---

63. if (!rdd.isEmpty()) { 
    

    ---

64. ---

65. // temp table 
    

    ---

66. rdd.map(_.value.split("_")).map(p => Person(p(0), p(1).trim.toInt, p(2))).toDF.createOrReplaceTempView("temp") 
    

    ---

67. ---

68. // use spark sql 
    

    ---

69. val rs = spark.sql("select * from temp") 
    

    ---

70. ---

71. // create hbase conf 
    

    ---

72. val hconf = HBaseConfiguration.create 
    

    ---

73. hconf.set("hbase.zookeeper.quorum", "localhost"); //ZKFC 
    

    ---

74. hconf.set("hbase.zookeeper.property.clientPort", "2181") 
    

    ---

75. hconf.set("hbase.defaults.for.version.skip", "true") 
    

    ---

76. hconf.set(TableOutputFormat.OUTPUT_TABLE, "t1") // t1是表名， 表里面有一个列簇 cf1 
    

    ---

77. hconf.setClass("mapreduce.job.outputformat.class", classOf[TableOutputFormat[String]], classOf[OutputFormat[String, Mutation]]) 
    

    ---

78. val jobConf = new JobConf(hconf) 
    

    ---

79. ---

80. // convert every line to hbase lines 
    

    ---

81. rs.rdd.map(line => { 
    

    ---

82. val put = new Put(Bytes.toBytes(UUID.randomUUID().toString.substring(0, 9))) 
    

    ---

83. put.addColumn(Bytes.toBytes("cf1") 
    

    ---

84. , Bytes.toBytes("name") 
    

    ---

85. , Bytes.toBytes(line.get(0).toString) 
    

    ---

86. ) 
    

    ---

87. put.addColumn(Bytes.toBytes("cf1") 
    

    ---

88. , Bytes.toBytes("age") 
    

    ---

89. , Bytes.toBytes(line.get(1).toString) 
    

    ---

90. ) 
    

    ---

91. put.addColumn(Bytes.toBytes("cf1") 
    

    ---

92. , Bytes.toBytes("key") 
    

    ---

93. , Bytes.toBytes(line.get(2).toString) 
    

    ---

94. ) 
    

    ---

95. (new ImmutableBytesWritable, put) 
    

    ---

96. }).saveAsNewAPIHadoopDataset(jobConf) 
    

    ---

97. } 
    

    ---

98. }) 
    

    ---

99. ---

100. lines.map(record => record.value.split("_")).map(x=> (x(0),x(1), x(2))).print() 
     

     ---

101. ---

102. ssc start() 
     

     ---

103. ssc awaitTermination() 
     

     ---

104. ---

105. } 
     

     ---

106. } 
     

     ---

107. ---