Kafka连接SparkStreaming的两种方式

第一种方式代码:

 import org.apache.spark.storage.StorageLevel
 import org.apache.spark.{HashPartitioner, SparkConf}
 import org.apache.spark.streaming.kafka.KafkaUtils
 import org.apache.spark.streaming.{Seconds, StreamingContext}

 object KafkaWordCount {
   val updateFunc = (iter: Iterator[(String, Seq[Int], Option[Int])]) => {
     //iter.flatMap(it=>Some(it._2.sum + it._3.getOrElse(0)).map(x=>(it._1,x)))
     iter.flatMap { case (x, y, z) => Some(y.sum + z.getOrElse(0)).map(i => (x, i)) }
   }

   def main(args: Array[String]) {
     LoggerLevels.setStreamingLogLevels()
     val Array(zkQuorum, group, topics, numThreads) = args
     val sparkConf = new SparkConf().setAppName("KafkaWordCount").setMaster("local[2]")
     val ssc = new StreamingContext(sparkConf, Seconds(5))
     ssc.checkpoint("c://ck2")
     //"alog-2016-04-16,alog-2016-04-17,alog-2016-04-18"
     //"Array((alog-2016-04-16, 2), (alog-2016-04-17, 2), (alog-2016-04-18, 2))"
     val topicMap = topics.split(",").map((_, numThreads.toInt)).toMap
     val data = KafkaUtils.createStream(ssc, zkQuorum, group, topicMap, StorageLevel.MEMORY_AND_DISK_SER)
     val words = data.map(_._2).flatMap(_.split(" "))
     val wordCounts = words.map((_, 1)).updateStateByKey(updateFunc, new HashPartitioner(ssc.sparkContext.defaultParallelism), true)
     wordCounts.print()//老师给的代码文件中没有这句话  必须要有一个Action,否则报错
     //java.lang.IllegalArgumentException: requirement failed: No output operations registered, so nothing to execute
     ssc.start()
     ssc.awaitTermination()
   }
 }

第二种方式代码:

 import kafka.serializer.StringDecoder
 import org.apache.log4j.{Level, Logger}
 import org.apache.spark.SparkConf
 import org.apache.spark.rdd.RDD
 import org.apache.spark.streaming.kafka.{KafkaManager, KafkaUtils}
 import org.apache.spark.streaming.{Seconds, StreamingContext}

 object DirectKafkaWordCount {

   /*  def dealLine(line: String): String = {
       val list = line.split(',').toList
   //    val list = AnalysisUtil.dealString(line, ',', '"')// 把dealString函数当做split即可
       list.get(0).substring(0, 10) + "-" + list.get(26)
     }*/

   def processRdd(rdd: RDD[(String, String)]): Unit = {
     val lines = rdd.map(_._2)
     val words = lines.map(_.split(" "))
     val wordCounts = words.map(x => (x, 1L)).reduceByKey(_ + _)
     wordCounts.foreach(println)
   }

   def main(args: Array[String]) {
     if (args.length < 3) {
       System.err.println(
         s"""
            |Usage: DirectKafkaWordCount <brokers> <topics> <groupid>
            |  <brokers> is a list of one or more Kafka brokers
            |  <topics> is a list of one or more kafka topics to consume from
            |  <groupid> is a consume group
            |
         """.stripMargin)
       System.exit(1)
     }

     Logger.getLogger("org").setLevel(Level.WARN)

     val Array(brokers, topics, groupId) = args

     // Create context with 2 second batch interval
     val sparkConf = new SparkConf().setAppName("DirectKafkaWordCount")
     sparkConf.setMaster("local[*]")
     sparkConf.set("spark.streaming.kafka.maxRatePerPartition", "5")
     sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")

     val ssc = new StreamingContext(sparkConf, Seconds(2))

     // Create direct kafka stream with brokers and topics
     val topicsSet = topics.split(",").toSet
     val kafkaParams = Map[String, String](
       "metadata.broker.list" -> brokers,
       "group.id" -> groupId,
       "auto.offset.reset" -> "smallest"
     )

     val km = new KafkaManager(kafkaParams)

     val messages = km.createDirectStream[String, String, StringDecoder, StringDecoder](
       ssc, kafkaParams, topicsSet)

     messages.foreachRDD(rdd => {
       if (!rdd.isEmpty()) {
         // 先处理消息
         processRdd(rdd)
         // 再更新offsets
         km.updateZKOffsets(rdd)
       }
     })

     ssc.start()
     ssc.awaitTermination()
   }
 }

关于第二种方式可以参考:

http://blog.csdn.net/ligt0610/article/details/47311771