spark 读取hive 计算后写入hive

package com.grady

import org.apache.spark.SparkConf
import org.apache.spark.sql.{DataFrame, Row, SparkSession}

object HiveTableToTable {

  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setAppName("StuToStu2")
    val spark: SparkSession = SparkSession.builder().config(conf).enableHiveSupport().getOrCreate()

    //tableToTable1(spark)
    tableToTable2(spark)
  }

  /**
   * spark sql 方式
   * @param spark
   */
  def tableToTable1(spark: SparkSession): Unit = {
    spark.sql("select * from jiang.student").show()
    spark.sql("create table if not exists jiang.student_male like jiang.student;")
    spark.sql("insert overwrite table jiang.student_male select * from jiang.student where sex = 'male'")
  }

  /**
   * 编程方式
   * @param spark
   */
  def tableToTable2(spark: SparkSession):Unit = {
    spark.sql("create table if not exists jiang.student_female like jiang.student")
    val dataFrame = spark.sql("select * from jiang.student")
    val femaleDataSet = dataFrame.where("sex = 'female'")
    // 有它和 case class Student 才能toDF，直接定义写成类文件不行
    import spark.implicits._
    val studentsDF = femaleDataSet.rdd.map( r =>
      Student(r(0).toString.toInt, r(1).toString, r(2).toString, r(3).toString.toInt, r(4).toString)
    ).map(s => {
      Student(s.id, s.name, s.sex, 18, "FemaleFt")
    }).toDF()
    studentsDF.write.mode("overwrite").insertInto("jiang.student_female")

// 方法二
//    val schema = SchemaType.getStudentSchema()
//    这里studentsRDD 需要转换成RDD[Row] 才可以使用
//    val femaleStudentDF = spark.createDataFrame(studentsRDD, schema)
  }
}
case class Student(id: Int, name: String, sex: String, age: Int, department: String)

执行：

spark-submit --master local[2] --num-executors 10 --class com.grady.HiveTableToTable /app/data/appdeploy/usehive1-1.0-SNAPSHOT.jar

日志：

hive> select * from student_female;
2       xiaochen        female  18      FemaleFt
Time taken: 2.838 seconds, Fetched: 1 row(s)