spark练习题

site1,user1,-- ::
site1,user2,-- ::
site1,user3,-- ::
site1,user3,-- ::
site2,user4,-- ::
site3,user5,-- ::
site3,user6,-- ::
site4,user7,-- ::
site4,user7,-- ::

现在要对最近7天的日志进行统计,统计结果格式如下,key(date(日期),hour(时间),site(网站))
value:(pv (访问次数),uv(独立访问人数,相同的访客id去重))
统计结果需要存至Hbase,使用spark写出伪代码

object tst {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("test").setMaster("local[3]")
    val sc = new SparkContext(conf);
    val curentDay = "2016-11-20"
    val format = "yyyy-MM-dd"
    val sdf = new SimpleDateFormat(format)
    val rdd =  sc.textFile("D:\\test\\spark\\笔试题\\siteuser.txt").map(t=>{
      val fileds = t.split(",")
      val date = fileds().split(" ")();
      val hour = fileds().split(" ")().substring(,);
      //site1,user1,2016-11-20 02
      (fileds(),fileds(),date,hour)
    }).filter(t=>{
      TimeUtils.intervalDays(curentDay,t._3,sdf,sdf) <
    }).cache()//如果不cache,在计算uvrdd时会重复计算之前的操作
    val pvrdd = rdd.groupBy(t=>(t._3,t._4,t._1)).map(t=>(t._1,t._2.size))
    println(pvrdd.collect().toBuffer)
    val uvrdd = rdd.groupBy(t=>t).map(t=>((t._1._3,t._1._4,t._1._1),t._2.size)).reduceByKey(_+_);
    print(uvrdd.collect().toBuffer)

    sc.stop()
  }
}