sparksql 用反射的方式将rdd转换成dataset/dataframe

java

 public class ReflectionDemo {
     private static SparkConf conf = new SparkConf().setAppName("reflectdemo").setMaster("local");
     private static JavaSparkContext jsc = new JavaSparkContext(conf);

     private static SparkSession session = new SparkSession(jsc.sc());

     public static void main(String[] args) throws AnalysisException {

         // rdd
         JavaRDD<String> rdd = jsc.textFile("./src/main/java/cn/tele/spark_sql/rdd2dataset/students.txt");

         // 封装rdd
         JavaRDD<Student> mapRdd = rdd.map(new Function<String, Student>() {

             private static final long serialVersionUID = 1L;

             @Override
             public Student call(String v1) throws Exception {
                 String[] fields = v1.split(",");
                 Student student = new Student(Integer.valueOf(fields[0]), fields[1], Integer.valueOf(fields[2]));
                 return student;
             }
         });

         // 通过反射的方式进行转换
         Dataset<Row> dataset = session.createDataFrame(mapRdd, Student.class);

         // 注册为临时信息表
         // dataset.registerTempTable("students");

         dataset.createOrReplaceTempView("studentsView");

         Dataset<Row> result = session.sql("select * from studentsView where age<=18");
         // result.show();

         // 把dataset转换成rdd
         JavaRDD<Student> javaRDD = result.javaRDD().map(new Function<Row, Student>() {

             private static final long serialVersionUID = 1L;

             @Override
             public Student call(Row v1) throws Exception {
                 return new Student(v1.getAs("id"), v1.getAs("name"), v1.getAs("age"));
             }
         });

         javaRDD.foreach(new VoidFunction<Student>() {

             private static final long serialVersionUID = 1L;

             @Override
             public void call(Student t) throws Exception {
                 System.out.println(t);
             }
         });
         session.stop();
         jsc.close();
     }
 }

scala

 case class Student(val id:Int,val name:String,val age:Int)

 object ReflectionDemo {
   def main(args: Array[String]): Unit = {
     val conf = new SparkConf().setAppName("reflectdemo").setMaster("local")

     val sc = new SparkContext(conf)

     val sqlContext = new SQLContext(sc)

     //导入隐式转换
     import sqlContext.implicits._

     //创建rdd
     val rdd = sc.textFile("./src/main/scala/cn/tele/spark_sql/rdd2dataframe/students.txt", 2)

     //转换rdd为dataframe
     val dataframe = rdd.map(line => {
       val arr = line.split(",");
       new Student(arr(0).trim().toInt, arr(1).trim(), arr(2).trim().toInt)
     }).toDF()

     //创临时视图
     dataframe.createOrReplaceTempView("students")

     //  dataframe.show()

     val df = sqlContext.sql("select * from students where age<=18")

     val newRdd = df.rdd.map(row => new Student(row.getAs[Int]("id"), row.getAs[String]("name"), row.getAs[Int]("age")))

     newRdd.foreach(println(_))
   }

 }