DataFrame WordCount

测试数据：

**
  * 使用DataFrame实现WordCount
  */
object DataFrameWordCount {
  def main(args: Array[String]): Unit = {

    val spark = SparkSession.builder().appName(this.getClass.getSimpleName).master("local").getOrCreate()
    import spark.implicits._
    val linesDF = spark.sparkContext.textFile("D:\\workspace\\test_data.txt").toDF("line")
    linesDF.show(false)
    linesDF.printSchema()
    //将一行数据展开
    val wordsDF = linesDF.explode("line", "word")((line: String) => line.split(" "))
    wordsDF.printSchema()
    wordsDF.show(,false)
    //对 "word"列进行聚合逻辑并使用count算子计算每个分组元素的个数
    val wordCoungDF = wordsDF.groupBy("word").count()
    wordCoungDF.show(false)
    wordCoungDF.printSchema()
    println(wordCoungDF.count() + "----------")
  }

}

打印结果：

+------------+
|line        |
+------------+
|      |
|     |
|     |
|      |
+------------+

root
 |-- line: string (nullable = true)

root
 |-- line: string (nullable = true)
 |-- word: string (nullable = true)

+------------+----+
|line        |word|
+------------+----+
|      |   |
|      |   |
|      |   |
|      |   |
|      |   |
|      |   |
|     |   |
|     |   |
|     |   |
|     |   |
|     |   |
|     |  |
|     |  |
|     |   |
|     |   |
|     |   |
|     |   |
|     |   |
|      |   |
|      |   |
|      |   |
|      |   |
|      |   |
|      |   |
+------------+----+

+----+-----+
|word|count|
+----+-----+
|   |    |
|   |    |
|   |    |
|   |    |
|   |    |
|   |    |
|   |    |
|  |    |
|   |    |
|   |    |
+----+-----+

root
 |-- word: string (nullable = true)
 |-- count: long (nullable = false)