SparkSQLDemo.scala


import org.apache.spark.sql.{Row, SparkSession}
import org.apache.spark.sql.types.{StringType, StructField, StructType} object SparkSQLDemo { // $example on:create_ds$
case class Person(name: String, age: Long)
// $example on:create_ds$ def main(args: Array[String]): Unit = {
//开启SparkSession
// $example on: init_session$
val spark = SparkSession
.builder()
.appName("SparkSQLDemo")
.master("local")
.config("spark.some.config.option", "some-value")
.getOrCreate()
// $example off: init_session$ // runBasicDataFrameDemo(spark)
// runDatasetCreationDemo(spark)
// runInferSchemaDemo(spark)
runProgrammaticSchemaDemo(spark) //关闭SparkSeesion
spark.stop() } private def runBasicDataFrameDemo(spark: SparkSession) = { val df = spark.read.json("/Users/hadoop/app/spark/examples/src/main/resources/people.json") //Displays the content of the DataFrame to stdout
df.show() //Print the schema in a tree format
df.printSchema() //Select only the "name" column
df.select("name").show() //This import is needed to use the $-notation
import spark.implicits._
df.select($"name", $"age" + 1).show() //Select people older than 21
df.select($"age" > 21).show() //Count people by age
df.groupBy("age").count().show() //$example on: global_temp_view$
//Register the DataFrame as a SQL temporary view
df.createOrReplaceTempView("people")
val sqlDF = spark.sql("select * from people")
sqlDF.show() //Register the DataFrame as a global temporary view
df.createGlobalTempView("people") //Global temporary view is tied to a system preserved database `global_temp`
spark.sql("select * from global_temp.people").show //Global temporary view is cross-session
spark.newSession().sql("select * from global_temp.people").show() } private def runDatasetCreationDemo(spark: SparkSession) = { // A container for a [[Dataset]], used for implicit conversions in Scala.
// To use this, import implicit conversions in SQL:
import spark.implicits._ // .toDS() -> 这是用括号声明的,以防止Scala编译器将`rdd.toDS(“1”)`视为调用此toDS然后应用于返回的数据集。 //Encoder are created for case classes (为case class 创建编码器)
val caseClassDS = Seq(Person("Andy", 32)).toDS()
caseClassDS.show() //Encoders for most common types are automatically provided by importing spark.implicits._
val primitiveDS = Seq(1, 2, 3).toDS()
primitiveDS.map(_ + 1).foreach(println(_))//.collect() //DataFrames can be converted to a Dataset by providing a class. Mapping will bedone by name
val path = "/Users/hadoop/app/spark/examples/src/main/resources/people.json"
val peopleDS = spark.read.json(path).as[Person]
peopleDS.show() } private def runInferSchemaDemo(spark: SparkSession) = { // $example on: schema_inferring$
//For implicit conversions from RDDs to DataFrames
import spark.implicits._ //Create an RDD of Person objects from a text file, convert it to a DataFrame
val peopleDF = spark.sparkContext
.textFile("/Users/hadoop/app/spark/examples/src/main/resources/people.txt")
.map(_.split(","))
.map(x => Person(x(0), x(1).trim.toInt))
.toDF() //Register the DataFrame as a temporary view
peopleDF.createOrReplaceTempView("people") //SQL statements can be run by using the sql methods provided by Spark
val teenagersDF = spark.sql("select name, age from people where age between 13 and 19") //The columns of a row in the result can be accessed by field index
//(结果中的行的列可以通过字段索引访问)
teenagersDF.map(teenager => s"Name: ${teenager(0)}").show() //or by field name
teenagersDF.map(teenager => s"Name: ${teenager.getAs[String]("name")}").show() //No pre-defined encoders for Dataset[Map[K,V]], define explicitly
//(Dataset[Map[K,V]] 没有预定义的编码器, 显式定义)
implicit val mapEncoder = org.apache.spark.sql.Encoders.kryo[Map[String, Any]] //Primitive types and case classes can be also defined as
//(原始类型和case类也可以定义为隐式val )
//implicit val stringIntMapEncoder: Encoder[Map[String, Any]] = ExpressionEncoder() //row.getValuesMap[T] retrieves multiple columns at once into a Map[String, T]
teenagersDF.map(teenager =>
teenager.getValuesMap[Any](List("name", "age"))
).foreach(println(_))//.collect() // $example off: schema_inferring$ } private def runProgrammaticSchemaDemo(spark: SparkSession) = { import spark.implicits._
// $example on: programmatic_schema$ //Create an RDD
val peopleRDD = spark.sparkContext.textFile("/Users/hadoop/app/spark/examples/src/main/resources/people.txt") //The schema is encoded in a string
val schemaString = "name age" //Generate the schema based on the string of schema
val fields = schemaString.split(" ")
.map(fieldName => StructField(fieldName, StringType, nullable = true))
val schema = StructType(fields) //Convert records of the RDD (people) to Rows
val rowRDD = peopleRDD
.map(_.split(","))
.map(attributes => Row(attributes(0), attributes(1).trim)) //Apply the schema to the RDD
val peopleDF = spark.createDataFrame(rowRDD, schema) //Creates a temporary view using the DataFrame
peopleDF.createOrReplaceTempView("people") //SQL can be run over a temporary view created using DataFrames
val results = spark.sql("select name from people") //The results of SQL queries are DataFrames and support all the normal RDD operations
//The columns of a row in the result can be accessed by field index or by field name
results.map(attributes => s"Name: ${attributes(0)}").show() // $exmaple off: programmatic_schema$
}
}


sparkSQL中的example学习(1)的更多相关文章

  1. sparkSQL中的example学习(3)

    UserDefinedTypedAggregation.scala(用户可自定义类型) import org.apache.spark.sql.expressions.Aggregator impor ...

  2. sparkSQL中的example学习(2)

    UserDefinedUntypedAggregate.scala(默认返回类型为空,不能更改) import org.apache.spark.sql.{Row, SparkSession} imp ...

  3. PHP中的Libevent学习

    wangbin@2012,1,3 目录 Libevent在php中的应用学习 1.      Libevent介绍 2.      为什么要学习libevent 3.      Php libeven ...

  4. JS中childNodes深入学习

    原文:JS中childNodes深入学习 <html xmlns="http://www.w3.org/1999/xhtml"> <head> <ti ...

  5. CNCC2017中的深度学习与跨媒体智能

    CNCC2017中的深度学习与跨媒体智能 转载请注明作者:梦里茶 目录 机器学习与跨媒体智能 传统方法与深度学习 图像分割 小数据集下的深度学习 语音前沿技术 生成模型 基于贝叶斯的视觉信息编解码 珠 ...

  6. 【Spark篇】---SparkSQL中自定义UDF和UDAF,开窗函数的应用

    一.前述 SparkSQL中的UDF相当于是1进1出,UDAF相当于是多进一出,类似于聚合函数. 开窗函数一般分组取topn时常用. 二.UDF和UDAF函数 1.UDF函数 java代码: Spar ...

  7. 图解BERT(NLP中的迁移学习)

    目录 一.例子:句子分类 二.模型架构 模型的输入 模型的输出 三.与卷积网络并行 四.嵌入表示的新时代 回顾一下词嵌入 ELMo: 语境的重要性 五.ULM-FiT:搞懂NLP中的迁移学习 六.Tr ...

  8. python中confIgparser模块学习

    python中configparser模块学习 ConfigParser模块在python中用来读取配置文件,配置文件的格式跟windows下的ini配置文件相似,可以包含一个或多个节(section ...

  9. Scala中的类学习

    Scala中的类学习 从java了解类的情况下,了解Scala的类并不难.Scala类中的字段自动带getter和setter方法,用@BeanProperty注解生成javaBean对象的getXX ...

随机推荐

  1. Mysql—修改用户密码(重置密码)

    1.登录mysql [root@localhost ~]# mysql -uroot -p123456 [root@localhost ~]# mysql -hlocalhost -uroot -p1 ...

  2. Linux:挂载磁盘分区

    查看挂载的分区 df 命令主要用来了解系统中已经挂载的各个文件系统的磁盘使用情况. 常用选项: "-h" ,显示更易读的容量单位: "-T" ,显示文件系统的类 ...

  3. C++ 基础语法 快速复习笔记(3)---重载函数,多态,虚函数

    1.重载运算符和重载函数: C++ 允许在同一作用域中的某个函数和运算符指定多个定义,分别称为函数重载和运算符重载. 重载声明是指一个与之前已经在该作用域内声明过的函数或方法具有相同名称的声明,但是它 ...

  4. HTML 中img标签不显示

    异常 图片请求响应吗: 403, url响应如下: Request Method: GET Status Code: 403 Forbidden Remote Address: ***.***.**. ...

  5. 谷歌浏览器安装JsonView插件

    可方便阅读json格式文件,参考https://www.jianshu.com/p/6ea9f2245f4d

  6. scanf函数和cin的区别、类的数组、C++排序函数

    给定n个字符串,将这n个字符串按照字典序进行排列,此处用排列函数是C++的库函数sort,产生如下两个疑问,望大佬解答 #include <iostream> #include <a ...

  7. RFM模型的应用 - 电商客户细分(转)

    RFM模型是网点衡量当前用户价值和客户潜在价值的重要工具和手段.RFM是Rencency(最近一次消费),Frequency(消费频率).Monetary(消费金额) 消费指的是客户在店铺消费最近一次 ...

  8. 洛谷 P5596 【XR-4】题

    洛谷 P5596 [XR-4]题 洛谷传送门 题目描述 小 X 遇到了一道题: 给定自然数 a,ba,b,求满足下列条件的自然数对 (x,y)(x,y) 的个数: y^2 - x^2 = ax + b ...

  9. SpringMVC+ajax文件上传实例教程

    原文地址:https://blog.csdn.net/weixin_41092717/article/details/81080152 文件上传文件上传是项目开发中最常见的功能.为了能上传文件,必须将 ...

  10. mybatis批处理数据

    批处理数据主要有三种方式: 1.传统jdbc处理    2.mybatis批处理插入    3.使用executortype处理 jdbc 处理 1.通过 for循环插入 main方法如下所示: Co ...