spark_learn

package chapter03

import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by chenzechao on 2017/12/21.
  */

/**
  spark-shell                                      \
  --master yarn-client                             \
  --driver-memory 1G                               \
  --driver-cores 1                                 \
  --queue root.queue_0101_04                       \
  --executor-memory 2G                            \
  --num-executors 2                                \
  --conf spark.executor.cores=1                    \
  --name 'tmp_abc_test'                           \
  --conf spark.yarn.executor.memoryOverhead=4096   \
  --conf spark.driver.maxResultSize=8G             \
  --conf spark.sql.hive.metastore.version=1.2.1    \
  --conf spark.sql.shuffle.partitions=150
  */

object document {
  // 0 获取参数flag

  //0.设置环境
  val conf = new SparkConf().setAppName("tianchi").setMaster("local[*]")
  val sc = new SparkContext(conf)
  val sqlContext = new org.apache.spark.sql.SQLContext(sc)
  val hiveContext = new HiveContext(sc)

  val jsonFile = "file:///tmp/upload/data/json_file"
  val jsonFile_hdfs = "/tmp/ccc/tmpc/json_file"
  // 执行SQL
  val df1 = sqlContext.sql("select * from sx_360_safe.sub_ladm_exc_app_s16_all_for_double").limit(200).cache()
  df1.count()

  // Print the schema in a tree format
  df1.printSchema()

  // Select only then "gu_flag" column
  df1.select("gu_flag").show()

  // Select everybody, but increment the age by 1
  df1.select(df1("empno"),df1("age"),df1("age") + 1 ).show

  // Select emp age older than 21
  df1.filter(df1("age") > 21).select(df1("empno"),df1("age")).show()

  // Count emp by age
  df1.groupBy(df1("age")).count().sort(df1("age")).show()
  val gb = df1.groupBy(df1("age")).count()
  gb.sort(gb("count")).show()

  // save dataFrame as json file
  df1.write.mode("Overwrite").format("json").save(jsonFile_hdfs)
  df1.write.mode("Append").format("json").save(jsonFile_hdfs)
  df1.select(df1("empno"), df1("gu_flag")).write.mode("Overwrite").format("parquet").saveAsTable("sx_360_safe.tmp_czc_20180323_04")

  // this is used to implicitly convert an RDD to a DataFrame.
  import sqlContext.implicits._

  val df2 = sqlContext.read.json(jsonFile)

  // Encoders for most common types are automatically provided by importing sqlContext.implicits._
  val ds1 = Seq(1, 2, 3).toDS()
  ds1.map(_ + 1).collect()

  // Encoders are also created for case class
  case class Person(name:String ,age: Long)
  val ds = Seq(Person("Andy",35)).toDS()
  ds.show()

  /**
    * Inferring the Schema Using Reflection
    */
  import sqlContext.implicits._
  case class Person2(name:String, age:Int)
  val people = sc.textFile("/tmp/ccc/data/tmpa").filter(_.length > 1).map(_.split(",")).map(p => Person2(p(0),p(1).trim.toInt)).toDF()
  people.registerTempTable("people")
  sqlContext.sql("select * from people limit 10").show

  val teenagers = sqlContext.sql("select name,age from people where age >= 23 and age<= 26")
  teenagers.map(t => "Name: " + t(0)).collect().foreach(println)

  // or by field name
  teenagers.map(t => "Name: " + t.getAs[String]("name")).collect().foreach(println)

// row.getValuesMap[T] retrieves multiple columns at once into a Map[String,T]
  teenagers.map(_.getValuesMap[Any](List("name","age"))).collect().foreach(println)

  /**
    * Programmatically Specifying the Schema
    */
  val schemaString  = "name age"
  import org.apache.spark.sql.Row
  import org.apache.spark.sql.types.{StructType,StructField,StringType}

  val schema =
    StructType(
      schemaString.split(" ").map(fieldName => StructField(fieldName,StringType,true))
  )

  // Convert records of the RDD (people) to Rows
  val people2 = sc.textFile("/tmp/ccc/data/tmpa")
  val rowRDD = people2.map(_.split(",")).map(p => Row(p(0),p(1).trim))

  // Apply the schema to the RDD
  val peopleDataFrame = sqlContext.createDataFrame(rowRDD,schema)

  // Register the DataFrames as a table
  peopleDataFrame.registerTempTable("people")

  // SQL

  val df = sqlContext.read.load("/tmp/examples/src/main/resources/users.parquet")

  val df3 = sqlContext.read.format("json").load("/tmp/examples/src/main/resources/people.json")

  // Run SQL on files directly
  val df4 = sqlContext.sql("select * from parquet.`/tmp/examples/src/main/resources/users.parquet`")

  // Save modes
  /**
    * ErrorIfExists (default)
    * Append
    * Overwrite
    * Ignore
    */

  val parquetFile = sqlContext.read.parquet("")

}