分享一下spark streaming与flume集成的scala代码。

文章来自：http://www.cnblogs.com/hark0623/p/4172462.html   转发请注明

object LogicHandle {
  def main(args: Array[String]) {
    //添加这个不会报执行错误
    val path = new File(".").getCanonicalPath()
    System.getProperties().put("hadoop.home.dir", path);
    new File("./bin").mkdirs();
    new File("./bin/winutils.exe").createNewFile();

    //val sparkConf = new SparkConf().setAppName("SensorRealTime").setMaster("local[2]")
    val sparkConf = new SparkConf().setAppName("SensorRealTime")

    val ssc = new StreamingContext(sparkConf, Seconds(20))

    val hostname = "localhost"
    val port = 2345
    val storageLevel = StorageLevel.MEMORY_ONLY
    val flumeStream = FlumeUtils.createStream(ssc, hostname, port, storageLevel)

    val lhc = new LogicHandleClass();

    //日志格式化模板
    val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    val sdfHour = new SimpleDateFormat("HH");
    val sdfMinute = new SimpleDateFormat("mm")

    //存储数据的hash对象  key/value存储  根据文档规则，使用各统计指标的key/value
    var redisMap = new HashMap[String, String]

　　
　　　　
    flumeStream.foreachRDD(rdd => {
      val events = rdd.collect()
      //println("event count:" + events.length)
      var i = 1
      for (event <- events) {
        val sensorInfo = new String(event.event.getBody.array()) //单行记录
        //单行记录格式化
        val arrayFileds = sensorInfo.split(",")
        if (arrayFileds.length == 6) {
          val shopId = arrayFileds(0) //店内编号

          val floorId = shopId.substring(0, 5) //楼层编号
          val mac = arrayFileds(1)
          val ts = arrayFileds(2).toLong //时间戳
          val time = sdf.format(ts * 1000)
          var hour = sdfHour.format(ts * 1000)
          var minute = sdfMinute.format(ts * 1000)
          var allMinute = hour.toInt * 60 + minute.toInt

          val x = arrayFileds(3)
          val y = arrayFileds(4)
          val level = arrayFileds(5)

          //后边就是我的业务代码了，省略了
        }
      }

      //存储至redis中
      lhc.SetAll(redisMap)
    })

    ssc.start()
    ssc.awaitTermination()
  }
}