Spark SQL实战

一、程序

 package sparklearning

 import org.apache.log4j.Logger
 import org.apache.spark.SparkConf
 import org.apache.spark.SparkContext
 import org.apache.spark.sql.SQLContext
 import org.apache.spark.storage.StorageLevel
 import org.apache.log4j.Level

 object OnLineTradeStatistics {

   case class User(userID:String,gender:String,age:Int,registerDate:String,provice:String,career:String)
   case class TradeDetail(tradeID:String, tradeDate:String,productID:Int,amount:Int,userID:String)
   def main(args: Array[String]){

     //关闭不必要的日志显示
       Logger.getLogger("org.apache.hadoop").setLevel(Level.ERROR)
       Logger.getLogger("org.apache.spark").setLevel(Level.ERROR)
       Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)

       //设置应用程序
       val conf=new SparkConf().setAppName("On Line Trade Data").setMaster("local")
       val ctx=new SparkContext(conf)
       val sqlCtx=new SQLContext(ctx)
       import sqlCtx.implicits._

       //读文件  RDD-->DataFrame
       val userDF= ctx.textFile("/home/hadoop/data/on_line_trade_user.txt").map(_.split(" ")).map(u=>User(u(0),u(1),u(2).toInt,u(3),u(4),u(5))).toDF()
       userDF.registerTempTable("user")
       userDF.persist(StorageLevel.MEMORY_ONLY_SER)

       val tradeDF= ctx.textFile("/home/hadoop/data/on_line_trade_detail.txt").map(_.split(" ")).map(u=>TradeDetail(u(0),u(1),u(2).toInt,u(3).toInt,u(4))).toDF()
       tradeDF.registerTempTable("trade")//生成临时表
       tradeDF.persist(StorageLevel.MEMORY_ONLY_SER)

       val countOfTrade2016 = sqlCtx.sql("SELECT * FROM trade where tradeDate like '2016%'").count()
       println("2016 total money: "+countOfTrade2016)
   }
 }

二、结果