RDDTest.scala

/**
 * Created by root on 9/7/15.
 */
import org.apache.spark.SparkContext
import org.apache.spark.SparkConf

object RDDTest {
  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("RDDTest").setMaster("local")
    val sc = new SparkContext(conf)
    val lines = sc.textFile("/home/slh/data/rddtest.txt")
    //count the word
    val lineLengths = lines.map(s => s.length)  //rdd
    val totalLength = lineLengths.reduce((a, b) => a + b)
    println("total length: " + totalLength)

    //get the word count
    val word_count = lines.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey((a, b) => a + b)
    //word_count.saveAsTextFile("/home/slh/data/rddresult0")

    //get the sum
    val sum = lines.flatMap(line => line.split(" ")).map(word => (1, word)).reduceByKey((a, b) => a + b)
    //sum.saveAsTextFile("/home/slh/data/rddresult1")
    //the result is (1,3343566777879717727)
    //println("sum: " + sum)

    //accumulator
    val accum = sc.accumulator(0, "My Accumulator")
    sc.parallelize(Array(1,2,3,4)).foreach(x => accum += x)
    println("Accumulator of Array(1,2,3,4) : " + accum.value)
  }
}