Scala 练习题 学生分数案例

一、相关信息
题目：
1、统计班级人数
2、统计学生的总分
3、统计总分年级排名前十学生各科的分数
4、统计总分大于年级平均分的学生
5、统计每科都及格的学生
6、统计偏科最严重的前100名学生
数据样例（部分数据）：
1.学生信息数据：students.txt

1500100001,施笑槐,22,女,文科六班
1500100002,吕金鹏,24,男,文科七班
1500100003,单乐蕊,22,女,理科六班
1500100004,葛德曜,24,男,理科三班
1500100005,宣谷芹,22,女,理科五班
1500100006,边昂雄,21,男,理科二班
1500100007,尚孤风,23,女,文科六班
1500100008,符半双,22,女,理科六班
1500100009,沈德昌,21,男,理科一班
1500100010,羿彦昌,23,男,理科六班
1500100011,宰运华,21,男,理科三班
1500100012,梁易槐,21,女,理科一班
1500100013,逯君昊,24,男,文科二班
1500100014,羿旭炎,23,男,理科五班
1500100015,宦怀绿,21,女,理科一班
1500100016,潘访烟,23,女,文科一班

2.学生分数信息（部分）：

500100001,1000001,98
1500100001,1000002,5
1500100001,1000003,0
1500100001,1000004,29
1500100001,1000005,85
1500100001,1000006,52
1500100002,1000001,139
1500100002,1000002,102
1500100002,1000003,44
1500100002,1000004,18
1500100002,1000005,46
1500100002,1000006,91
1500100003,1000001,48

3.学生科目信息（部分）：

1000001,语文,150
1000002,数学,150
1000003,英语,150
1000004,政治,100
1000005,历史,100
1000006,物理,100
1000007,化学,100
1000008,地理,100
1000009,生物,100

 

二、题目代码编写

1、统计班级人数

package shujia

import scala.io.Source
//1、统计班级人数
/**
 * 以下所有的方法都是返回新的集合，不会修改原始的集合
 * 同时以下这些方法在set集合中也有，除了sort
 * foreach:遍历数据
 * map：一条一条处理数据
 * filter：过滤数据
 * flatMap:将一行转换成多行
 * sortBy：排序
 * groupBy：分组
 */
object Test2 {
  def main(args: Array[String]): Unit = {
    //读取文件
    val students: List[String] = Source.fromFile("data/score.txt").getLines().toList
    //按照逗号分割

    val stringses: List[Array[String]] = students.map(line => line.split(","))

    //3.过滤脏数据
    val listFilter: List[Array[String]] = stringses.filter(line => line.length == 3)

    //4.取数据
    val scores: List[(String, Int)] = listFilter.map {
      case Array(id: String, _: String, sco: String) =>
        (id, sco.toInt)
    }
    //分组group
    val group: Map[String, List[(String, Int)]] = scores.groupBy(word => word._1)
    //统计数量

    val sumScoList: Map[String, Int] = group.map {
      case (id: String, list: List[(String, Int)]) =>
        val sco: List[Int] = list.map { case (_, sco: Int) => sco }
        val sumSco: Int = sco.sum
        (id, sumSco)
    }
    sumScoList.foreach(println)
  }
}

2、统计学生的总分

package com.shujia.scala

import scala.io.Source

object Demo22SumScore {
  def main(args: Array[String]): Unit = {
    /**
     * 2、统计学生的总分
     */

    //1、读取分数表
    val scoresList: List[String] = Source.fromFile("data/score.txt").getLines().toList

    //2、过滤脏数据
    val filterList: List[String] = scoresList.filter((line: String) => {
      val length: Int = line.split(",").length
      length == 3
    })

    //3、取出学号和分数
    val idAndScore: List[(String, Int)] = filterList.map((line => {
      val split: Array[String] = line.split(",")
      //学号
      val id: String = split.head
      //分数
      val score: Int = split.last.toInt
      (id, score)
    }))

    //4、按照学号分组
    val groupByList: Map[String, List[(String, Int)]] = idAndScore.groupBy(kv => kv._1)

    //5、统计学生的总分
    val sumScoMap: Map[String, Int] = groupByList.map((kv: (String, List[(String, Int)])) => {
      val id: String = kv._1
      val scores: List[(String, Int)] = kv._2
      //取出每个学生所有的分数
      val scos: List[Int] = scores.map(sco => sco._2)
      //计算总分
      val sumSco: Int = scos.sum

      (id, sumSco)
    })

    sumScoMap.foreach(println)
  }

}
/*
* 第二种方法，case
*/

object Demo22SumScore {
  def main(args: Array[String]): Unit = {
    /**
     * 2、统计学生的总分
     */

    //1、读取分数表
    val scoresList: List[String] = Source.fromFile("data/score.txt").getLines().toList

    //2、过滤脏数据
    val filterList: List[String] = scoresList.filter((line: String) => {
      val length: Int = line.split(",").length
      length == 3
    })

    //3、取出学号和分数
    val idAndScore: List[(String, Int)] = filterList.map((line => {
      val split: Array[String] = line.split(",")
      //学号
      val id: String = split.head
      //分数
      val score: Int = split.last.toInt
      (id, score)
    }))

    //4、按照学号分组
    val groupByList: Map[String, List[(String, Int)]] = idAndScore.groupBy(kv => kv._1)

    //5、统计学生的总分
    val sumScoMap: Map[String, Int] = groupByList.map((kv: (String, List[(String, Int)])) => {
      val id: String = kv._1
      val scores: List[(String, Int)] = kv._2
      //取出每个学生所有的分数
      val scos: List[Int] = scores.map(sco => sco._2)
      //计算总分
      val sumSco: Int = scos.sum

      (id, sumSco)
    })

    sumScoMap.foreach(println)
  }

}

3、统计总分年级排名前十学生各科的分数

package shujia

import scala.io.Source

//1、统计总分年级排名前十学生各科的分数
object Test3Top10 {
  def main(args: Array[String]): Unit = {

    //1、读取分数
    val lines: List[String] = Source.fromFile("data/score.txt").getLines().toList

    //2、切分数据
    val scoreArr: List[Array[String]] = lines.map(line => line.split(","))

    //3、过滤脏数据
    val scoreFilter: List[Array[String]] = scoreArr.filter(arr => arr.length == 3)
//scoreFilter.foreach(println)
    //4、取出学号和分数
    val scoFilter: List[(String, String, Int)] = scoreFilter.map {
      case Array(id: String, subject: String, sco: String) =>
        (id, subject, sco.toInt)
    }
    //5.学号分组
    val scoGroupBy: Map[String, List[(String, String, Int)]] = scoFilter.groupBy(kv => kv._1)

    //6.计算学生总分
    val sSos: List[(String, Int, List[(String, String, Int)])] = scoGroupBy.map {
      case (id: String, list: List[(String, String, Int)]) =>
        val scores: List[Int] = list.map { case (_, _, sco: Int) => sco }

        val scoSum: Int = scores.sum
        (id, scoSum, list)
    }.toList
    val lists: List[(String, Int, List[(String, String, Int)])] = sSos.sortBy(kv => -kv._2)
    val top10: List[(String, Int, List[(String, String, Int)])] = lists.take(10)
    top10.foreach(println)
  }
}

4、统计总分大于年级平均分的学生

import com.shujia.spark.util.HdfsUtil
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

//统计总分大于年级平均分的学生
//平均总分=学生总分/学生人数
object Test2ScoreAvg {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
    conf.setAppName("AVG")
    //    conf.setMaster("local")
    val sc = new SparkContext(conf)

    //读取文件切分过滤脏数据
    val scoFilter: RDD[Array[String]] = sc.textFile("/data/student/score.txt").map(_.split(",")).filter(_.length == 3)
    //提取分数出来
    val scoRDD: RDD[(String, Int)] = scoFilter.map {
      case Array(id: String, _, sco: String) =>
        (id, sco.toInt)
    }
    //按照学生学号进行分组
    val scoGroRDD: RDD[(String, Iterable[(String, Int)])] = scoRDD.groupBy(_._1)
    //暂存缓存中，提高速率
    scoRDD.cache()
    //计算学生的总分
    val sumStusRDD: RDD[(String, Int)] = scoRDD.reduceByKey(_ + _)
    //计算年级的总分
    val sumNJ: Double = sumStusRDD.map(_._2).sum
    //计算年级的平均总分
    val avgSum: Double = sumNJ / scoGroRDD.count
    //过滤总分大于平均分的数据
    val avgtoSum: RDD[(String, Int)] = sumStusRDD.filter { case (_, sco: Int) => sco >= avgSum }
    val l1: Long = avgtoSum.count()
    println(s"大于平均分有$l1+人,平均分是：$avgSum")
    //    avgtoSum.foreach(println)
    HdfsUtil.delete("/data/sum_avgToSum")
    avgtoSum.saveAsTextFile("/data/sum_avgToSum")
  }

}

5、统计每科都及格的学生

package shujia

import scala.collection.immutable
import scala.io.Source

//3、统计每科都及格的学生
object Test5_60fen {
  def main(args: Array[String]): Unit = {
    //读取文件
    val list: List[String] = Source.fromFile("data/score.txt").getLines().toList
    //    list.foreach(println)
    //按照逗号分割
    val listSplit: List[Array[String]] = list.map(line => line.split(","))
    //    listSplit.foreach(println)
    //过滤数据
    val listFilter: List[Array[String]] = listSplit.filter(line => line.length == 3)
    //提取数据,过滤分数大于60的人
//    listFilter.foreach(println)
    val listFilter2: List[Array[String]] = listFilter.filter(sco => sco.last.toInt - (60) >= 0)
    //listFilter2.foreach(println)
    val lists: List[(String, String, Int)] = listFilter2.map {
      case Array(id: String, sub: String, sco: String) =>
        (id, sub, sco.toInt)
    }
    //按照学号分组
    val listGroup: Map[String, List[(String, String, Int)]] = lists.groupBy(line => line._1)

    val list1: List[(String, List[(String, String, Int)])] = listGroup.map((kv: (String, List[(String, String, Int)])) => {
      val id: String = kv._1
      val count: List[(String, String, Int)] = kv._2

      (id, count)
    }).toList

       list1.foreach(println)
  }
}

6、统计偏科最严重的前100名学生

package com.shujia.scala

import scala.collection.immutable
import scala.io.Source

object Demo31Student {
  def main(args: Array[String]): Unit = {
    /**
     * 4、统计偏科最严重的前100名学生
     *
     * 偏科评估的标准： 方差
     */

    //1、读取分数
    val lines: List[String] = Source.fromFile("data/score.txt").getLines().toList

    //2、切分数据
    val scoreArr: List[Array[String]] = lines.map(line => line.split(","))

    //3、过滤脏数据
    val scoreFilter: List[Array[String]] = scoreArr.filter(arr => arr.length == 3)

    //4、取出学号和分数
    val idAndScore: List[(String, Int)] = scoreFilter.map {
      case Array(id: String, _, sco: String) =>
        (id, sco.toInt)
    }

    //5、按照学号分组
    val groupBy: Map[String, List[(String, Int)]] = idAndScore.groupBy(kv => kv._1)

    //计算方差
    val std: List[(String, Double, List[(String, Int)])] = groupBy.map {
      case (id: String, list: List[(String, Int)]) =>
        //一个学生所有的分数
        val scores: List[Int] = list.map { case (_, sco: Int) => sco }

        /**
         * 计算方差
         * 1、计算总数
         * 2、计算平均值
         * 3、计算方差
         *
         */
        //科目数
        val N: Double = scores.length.toDouble
        //平均数
        val avg: Double = scores.sum / N

        //计算方差
        val std: Double = scores.map((sco: Int) => (sco - avg) * (sco - avg)).sum / N

        (id, std, list)
    }.toList

    //按照方差排序，取前100
    val sortByStd: List[(String, Double, List[(String, Int)])] = std.sortBy(kv => -kv._2)

    //取前100
    val top10: List[(String, Double, List[(String, Int)])] = sortByStd.take(100)

    top10.foreach(println)
  }

}