sparksql parquet 合并元数据

java

 public class ParquetMergeSchema {
     private static SparkConf conf = new SparkConf().setAppName("parquetmergeschema").setMaster("local");
     private static JavaSparkContext jsc = new JavaSparkContext(conf);
     private static SparkSession session = new SparkSession(jsc.sc());

     public static void main(String[] args) {
         JavaRDD<Tuple2<String, Object>> rdd1 = jsc.parallelize(
                 Arrays.asList(new Tuple2<String, Object>("jack", 21), new Tuple2<String, Object>("lucy", 20)));

         JavaRDD<Row> row1 = rdd1.map(new Function<Tuple2<String, Object>, Row>() {

             private static final long serialVersionUID = 1L;

             @Override
             public Row call(Tuple2<String, Object> v1) throws Exception {
                 return RowFactory.create(v1._1, v1._2);
             }
         });

         JavaRDD<Tuple2<String, Object>> rdd2 = jsc.parallelize(
                 Arrays.asList(new Tuple2<String, Object>("jack", "A"), new Tuple2<String, Object>("yeye", "B")));

         JavaRDD<Row> row2 = rdd2.map(new Function<Tuple2<String, Object>, Row>() {

             private static final long serialVersionUID = 1L;

             @Override
             public Row call(Tuple2<String, Object> v1) throws Exception {
                 return RowFactory.create(v1._1, v1._2);
             }
         });

         StructType schema1 = DataTypes
                 .createStructType(Arrays.asList(DataTypes.createStructField("name", DataTypes.StringType, false),
                         DataTypes.createStructField("age", DataTypes.IntegerType, false)));

         StructType schema2 = DataTypes
                 .createStructType(Arrays.asList(DataTypes.createStructField("name", DataTypes.StringType, false),
                         DataTypes.createStructField("grade", DataTypes.StringType, false)

                 ));

         // 将rdd转成dataset
         Dataset<Row> ds1 = session.createDataFrame(row1, schema1);

         Dataset<Row> ds2 = session.createDataFrame(row2, schema2);

         // 保存为parquet文件
         ds1.write().mode(SaveMode.Append).save("./src/main/java/cn/tele/spark_sql/parquet/mergetest");
         ds2.write().mode(SaveMode.Append).save("./src/main/java/cn/tele/spark_sql/parquet/mergetest");

         // 指定parquet文件的目录进行读取,设置mergeSchema为true进行合并
         Dataset<Row> dataset = session.read().option("mergeSchema", true)
                 .load("./src/main/java/cn/tele/spark_sql/parquet/mergetest");

         dataset.printSchema();
         dataset.show();

         session.stop();
         jsc.close();

     }
 }

scala

 object ParquetMergeSchema {
   def main(args: Array[String]): Unit = {
     val conf = new SparkConf().setAppName("parquetmergeschema").setMaster("local")
     val sc = new SparkContext(conf)
     val sqlContext = new SQLContext(sc)

     val rdd1 = sc.parallelize(Array(("jack", 18), ("tele", 20)), 2).map(tuple => { Row(tuple._1, tuple._2) })
     val rdd2 = sc.parallelize(Array(("tele", "A"), ("wyc", "A"), ("yeye", "C")), 2).map(tuple => { Row(tuple._1, tuple._2) })

     //schema
     val schema1 = DataTypes.createStructType(Array(
       StructField("name", DataTypes.StringType, false),
       StructField("age", DataTypes.IntegerType, false)))

     val schema2 = DataTypes.createStructType(Array(
       StructField("name", DataTypes.StringType, false),
       StructField("grade", DataTypes.StringType, false)))

     //转换
     val df1 = sqlContext.createDataFrame(rdd1, schema1)
     val df2 = sqlContext.createDataFrame(rdd2, schema2)

     //写出
     df1.write.mode(SaveMode.Append).save("./src/main/scala/cn/tele/spark_sql/parquet/mergetest")
     df2.write.mode(SaveMode.Append).save("./src/main/scala/cn/tele/spark_sql/parquet/mergetest")

     //读取进行合并
     val df = sqlContext.read.option("mergeSchema", true).parquet("./src/main/scala/cn/tele/spark_sql/parquet/mergetest")
     df.printSchema()
     df.show()
   }
 }