Spark 覆盖写Hive分区表,只覆盖部分对应分区

要求Spark版本2.3以上，亲测2.2无效

配置

config("spark.sql.sources.partitionOverwriteMode","dynamic")

注意

1、saveAsTable方法无效，会全表覆盖写，需要用insertInto，详情见代码

2、insertInto需要主要DataFrame列的顺序要和Hive表里的顺序一致，不然会数据错误！

package com.dkl.blog.spark.hive

import org.apache.spark.sql.SparkSession

/**

• Created by dongkelun on 2020/1/16 15:25

• 博客：Spark 覆盖写Hive分区表,只覆盖部分对应分区

• 要求Spark版本2.3以上
  
  */
  
  object SparkHivePartitionOverwrite {
  
  def main(args: Array[String]): Unit = {
  
  val spark = SparkSession
  
  .builder()
  
  .appName("SparkHivePartitionOverwrite")
  
  .master("local")
  
  .config("spark.sql.parquet.writeLegacyFormat", true)
  
  .config("spark.sql.sources.partitionOverwriteMode","dynamic")
  
  .enableHiveSupport()
  
  .getOrCreate()

  import spark.sql

  val data = Array(("001", "张三", 21, "2018"), ("002", "李四", 18, "2017"))

  val df = spark.createDataFrame(data).toDF("id", "name", "age", "year")
  
  //创建临时表
  
  df.createOrReplaceTempView("temp_table")

  val tableName="test_partition"
  
  //切换hive的数据库
  
  sql("use test")
  
  // 1、创建分区表，并写入数据
  
  df.write.mode("overwrite").partitionBy("year").saveAsTable(tableName)

  spark.table(tableName).show()

  val data1 = Array(("011", "Sam", 21, "2018"))

  val df1 = spark.createDataFrame(data1).toDF("id", "name", "age", "year")
  
  // df1.write.mode("overwrite").partitionBy("year").saveAsTable(tableName) //不成功，全表覆盖
  
  // df1.write.mode("overwrite").format("Hive").partitionBy("year").saveAsTable(tableName) //不成功，全表覆盖
  
  df1.write.mode("overwrite").insertInto(tableName)

  spark.table(tableName).show()

  spark.stop
  
  }

}

结果

+---+----+---+----+

| id|name|age|year|

+---+----+---+----+

|002| 李四| 18|2017|

|001| 张三| 21|2018|

+---+----+---+----+

+---+----+---+----+

| id|name|age|year|

+---+----+---+----+

|011| Sam| 21|2018|

+---+----+---+----+