sparksql 动态设置schema将rdd转换成dataset/dataframe

java

 public class DynamicDemo {
     private static SparkConf conf = new SparkConf().setAppName("dynamicdemo").setMaster("local");
     private static JavaSparkContext jsc = new JavaSparkContext(conf);
     private static SparkSession session = new SparkSession(jsc.sc());

     public static void main(String[] args) {

         // 创建rdd
         JavaRDD<String> rdd = jsc.textFile("./src/main/java/cn/tele/spark_sql/rdd2dataset/students.txt");

         // 创建Row的rdd
         JavaRDD<Row> rowRdd = rdd.map(new Function<String, Row>() {

             private static final long serialVersionUID = 1L;

             @Override
             public Row call(String v1) throws Exception {
                 String[] fields = v1.split(",");
                 return RowFactory.create(Integer.valueOf(fields[0]), fields[1], Integer.valueOf(fields[2]));
             }
         });

         // 创建schema
         StructType schema = DataTypes
                 .createStructType(Arrays.asList(DataTypes.createStructField("id", DataTypes.IntegerType, false),
                         DataTypes.createStructField("name", DataTypes.StringType, false),
                         DataTypes.createStructField("age", DataTypes.IntegerType, false)));

         // 转换
         Dataset<Row> dataset = session.createDataFrame(rowRdd, schema);

         dataset.createOrReplaceTempView("students");

         Dataset<Row> result = session.sql("select * from students where age<=18");
         result.show();
 　　　　 session.stop();
         jsc.close();
     }
 }

scala

 object DynamicDemo {
   def main(args: Array[String]): Unit = {
     val conf = new SparkConf().setAppName("reflectdemo").setMaster("local")

     val sc = new SparkContext(conf)

     val sqlContext = new SQLContext(sc)

     //创建rdd
     val rdd = sc.textFile("./src/main/scala/cn/tele/spark_sql/rdd2dataframe/students.txt", 8)

     val rowRdd = rdd.map(lines => {
       val arr = lines.split(",");
       Row(arr(0).trim().toInt, arr(1), arr(2).trim().toInt)
     })

     val schema = DataTypes.createStructType(Array(
       /*    DataTypes.createStructField("id",DataTypes.IntegerType,false),
           DataTypes.createStructField("name",DataTypes.StringType,false),
           DataTypes.createStructField("age",DataTypes.IntegerType,false)*/
       StructField("id", DataTypes.IntegerType, false),
       StructField("name", DataTypes.StringType, false),
       StructField("age", DataTypes.IntegerType, false)))

     //转换
     val dataframe = sqlContext.createDataFrame(rowRdd, schema)

     dataframe.createOrReplaceTempView("students")

     val result = sqlContext.sql("select * from students where age<=18")
     result.show()
   }
 }