spark1.4的本地模式编程练习（1）

spark编程练习

申明：以下代码仅作学习参考使用，勿使用在商业用途。

• Wordcount
• UserMining
• TweetMining
• HashtagMining
• InvertedIndex
• Test

---

Test代码

package tutorial;

import java.util.Arrays;
import java.util.List;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;

public class Test {

    public static void main(String[] args) {
        // TODO Auto-generated method stub
        SparkConf conf = new SparkConf().setAppName("test").setMaster("spark://master:7077");
        @SuppressWarnings("resource")
        JavaSparkContext sc = new JavaSparkContext(conf);
//      sc.addJar("/home/sun/jars/myjar.jar");
        List<Integer> data = Arrays.asList(1, 2, 3, 4, 5);
        JavaRDD<Integer> distData = sc.parallelize(data);
        System.out.println(distData.count());
    }

}

两个Utils方法

1.解析json的代码

package utils;

import java.io.IOException;

import com.fasterxml.jackson.databind.ObjectMapper;

public class Parse {

  public static Tweet parseJsonToTweet(String jsonLine) {

    ObjectMapper objectMapper = new ObjectMapper();
    Tweet tweet = null;

    try {
      tweet = objectMapper.readValue(jsonLine, Tweet.class);
    } catch (IOException e) {
      e.printStackTrace();
    }
    return tweet;
  }
}

2.实体类

package utils;

import java.io.Serializable;

public class Tweet implements Serializable {

  long id;
  String user;
  String userName;
  String text;
  String place;
  String country;
  String lang;

  public String getUserName() {
    return userName;
  }

  public String getLang() {
    return lang;
  }

  public long getId() {
    return id;
  }

  public String getUser() { return user;}

  public String getText() {
    return text;
  }

  public String getPlace() {
    return place;
  }

  public String getCountry() {
    return country;
  }

  public void setId(long id) {
    this.id = id;
  }

  public void setUser(String user) {
    this.user = user;
  }

  public void setUserName(String userName) {
    this.userName = userName;
  }

  public void setText(String text) {
    this.text = text;
  }

  public void setPlace(String place) {
    this.place = place;
  }

  public void setCountry(String country) {
    this.country = country;
  }

  public void setLang(String lang) {
    this.lang = lang;
  }

  @Override
  public String toString(){
    return getId() + ", " + getUser() + ", " + getText() + ", " + getPlace() + ", " + getCountry();
  }
}

reduced-tweets.json

数据以及测试代码的获取请点击 这里。

WordCount代码块

package tutorial;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;

import java.util.Arrays;

/*
 *  step 1, the mapper:
 *
 *  -我们为每一个单词添加属性 1.获取形如（word,1）的 JavaPairRDD<String, Integer>。单词作为key
 *
 *  step 2, the reducer:
 *  -合并统计.
 *
 *
 */
public class Wordcount {

  private static String pathToFile = "data/wordcount.txt";

  public JavaRDD<String> loadData() {
    SparkConf conf = new SparkConf()
        .setAppName("Wordcount")
        .set("spark.driver.allowMultipleContexts", "true")
        .setMaster("local[*]"); // here local mode. And * means you will use as much as you have cores.

    JavaSparkContext sc = new JavaSparkContext(conf);

    JavaRDD<String> words = sc.textFile(pathToFile).flatMap(new FlatMapFunction<String, String>(){
        public Iterable call(String line) throws Exception {
            return Arrays.asList( line.split(" ")) ;
        }
    });

    return words;

  }

  /**
   *  Now count how much each word appears!
   */
  public JavaPairRDD<String, Integer> wordcount() {
    JavaRDD<String> words = loadData();

    // code here
    JavaPairRDD<String, Integer> couples = null;

    // code here
    JavaPairRDD<String, Integer> result = null;

    return result;
  }

  /**
   *  Now keep the word which appear strictly more than 4 times!
   */
  public JavaPairRDD<String, Integer> filterOnWordcount() {
    JavaPairRDD<String, Integer> wordcounts = wordcount();

    // TODO write code here
    JavaPairRDD<String, Integer> filtered = null;

    return filtered;

  }

}

UserMining代码块

package tutorial;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;

import utils.Parse;
import utils.Tweet;

/**
 * The Java Spark API documentation:
 * http://spark.apache.org/docs/latest/api/java/index.html
 *
 * 我们使用包含了8198个tweet数据记录。数据格式如下：
 *
 * {"id":"572692378957430785", "user":"Srkian_nishu :)", "text":
 * "@always_nidhi @YouTube no i dnt understand bt i loved of this mve is rocking"
 * , "place":"Orissa", "country":"India"}
 *
 * 目标：   找出user所有的tweet账户（一个user可能包含多个tweet账户，如Srkian_nishu的tweet账户有[572692378957430785，...]）
 *
 */
public class UserMining {

    private static String pathToFile = "data/reduced-tweets.json";

    public JavaRDD<Tweet> loadData() {
        // Create spark configuration and spark context
        SparkConf conf = new SparkConf().setAppName("User mining").set("spark.driver.allowMultipleContexts", "true")
                .setMaster("local[*]");

        JavaSparkContext sc = new JavaSparkContext(conf);

        // Load the data and parse it into a Tweet.
        // Look at the Tweet Object in the TweetUtils class.
        JavaRDD<Tweet> tweets = sc.textFile(pathToFile).map(new Function<String, Tweet>() {
            public Tweet call(String line) throws Exception {
                // TODO Auto-generated method stub
                return Parse.parseJsonToTweet(line);
            }

        });

        return tweets;
    }

    /**
     * For each user return all his tweets
     */
    public JavaPairRDD<String, Iterable<Tweet>> tweetsByUser() {
        JavaRDD<Tweet> tweets = loadData();

        // TODO write code here
        // Hint: the Spark API provides a groupBy method
        JavaPairRDD<String, Iterable<Tweet>> tweetsByUser = null;

        return tweetsByUser;
    }

    /**
     * Compute the number of tweets by user
     */
    public JavaPairRDD<String, Integer> tweetByUserNumber() {
        JavaRDD<Tweet> tweets = loadData();

        // TODO write code here
        // Hint: think about what you did in the wordcount example
        JavaPairRDD<String, Integer> count = null;

        return count;
    }

}

TweetMining代码块

package tutorial;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;

import scala.Tuple2;
import utils.Parse;
import utils.Tweet;

import java.io.Serializable;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * The Java Spark API documentation:
 * http://spark.apache.org/docs/latest/api/java/index.html
 ** 我们使用包含了8198个tweet数据记录。数据格式如下：
 *
 * {"id":"572692378957430785", "user":"Srkian_nishu :)", "text":
 * "@always_nidhi @YouTube no i dnt understand bt i loved of this mve is rocking"
 * , "place":"Orissa", "country":"India"}
 *
 * 目标： 1.找出所有被@的人
 *      2.计算每个人被@到的次数，找出前10个@次数最多的人
 *
 *
 * Use the TweetMiningTest to implement the code.
 */
public class TweetMining implements Serializable {

    /**
     *
     */

    private static String pathToFile = "data/reduced-tweets.json";

    /**
     * Load the data from the json file and return an RDD of Tweet
     */
    public JavaRDD<Tweet> loadData() {
        // create spark configuration and spark context
        SparkConf conf = new SparkConf().setAppName("Tweet mining").setMaster("spark://master:7077");
        conf.set("spark.driver.allowMultipleContexts" ,"true");
        JavaSparkContext sc = new JavaSparkContext(conf);
        sc.addJar("/home/sun/jars/tutorial-all.jar");

        // load the data and create an RDD of Tweet
        JavaRDD<Tweet> tweets = sc.textFile("hdfs://master:9000/sparkdata/reduced-tweets.json")
                .map(new Function<String, Tweet>() {
                    public Tweet call(String line) throws Exception {
                        // TODO Auto-generated method stub
                        return Parse.parseJsonToTweet(line);
                    }

                });
        return tweets;
    }

    /**
     * Find all the persons mentioned on tweets (case sensitive)
     */
    public JavaRDD<String> mentionOnTweet() {
        JavaRDD<Tweet> tweets = loadData();

        // You want to return an RDD with the mentions
        // Hint: think about separating the word in the text field and then find
        // the mentions
        // TODO write code here
        JavaRDD<String> mentions = tweets.flatMap(new FlatMapFunction<Tweet, String>() {
            public Iterable<String> call(Tweet t) throws Exception {
                String text = t.getText();
                Set<String> set = new HashSet<String>();
                String[] words = text.split(" ");
                for (String word : words) {
                    if (word.startsWith("@")) {
                        set.add(word);
                    }
                }
                return set;
            }

        });

        return mentions;

    }

    /**
     * Count how many times each person is mentioned
     */
    public JavaPairRDD<String, Integer> countMentions() {
        JavaRDD<String> mentions = mentionOnTweet();

        // Hint: think about what you did in the wordcount example
        // TODO write code here
        JavaPairRDD<String, Integer> mentionCount = mentions.mapToPair(new PairFunction<String, String, Integer>() {
            public Tuple2<String, Integer> call(String t) throws Exception {
                return new Tuple2<String, Integer>(t, 1);
            }
        }).reduceByKey(new Function2<Integer, Integer, Integer>() {
            public Integer call(Integer v1, Integer v2) throws Exception {
                // TODO Auto-generated method stub
                return v1 + v2;
            }
        });
        mentionCount.saveAsTextFile("hdfs://master:9000/sparkdata/tweets-m4");
        return mentionCount;
    }

    /**
     * Find the 10 most mentioned persons by descending order
     */
    public List<Tuple2<Integer, String>> top10mentions() {
        JavaPairRDD<String, Integer> counts = countMentions();

        // Hint: take a look at the sorting and take methods
        // TODO write code here
        List<Tuple2<Integer, String>> mostMentioned = null;

        return mostMentioned;
    }

    public static void main(String[] args) {
        Ex2TweetMining ex2TweetMining = new Ex2TweetMining();
        JavaPairRDD<String, Integer> res = ex2TweetMining.countMentions();
        System.out.println(res.take(1));
    }
}

HashtagMining代码块

package tutorial;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;

import scala.Tuple2;
import utils.Parse;
import utils.Tweet;

import java.util.List;

/**
 *  The Java Spark API documentation: http://spark.apache.org/docs/latest/api/java/index.html
 *
 * 我们使用包含了8198个tweet数据记录。数据格式如下：
 *
 * {"id":"572692378957430785", "user":"Srkian_nishu :)", "text":
 * "@always_nidhi @YouTube no i dnt understand bt i loved of this mve is rocking"
 * , "place":"Orissa", "country":"India"}
 *
 * 目标： 1.找出所有所有被标记（”#“）到的人。
 *      2.找出每个被标记（“#”）的人被（”@“）到的次数，求出次数前十
 *
 *
 */
public class HashtagMining {

  private static String pathToFile = "data/reduced-tweets.json";

  /**
   *  Load the data from the json file and return an RDD of Tweet
   */
  public JavaRDD<Tweet> loadData() {
    // create spark configuration and spark context
    SparkConf conf = new SparkConf()
        .setAppName("Hashtag mining")
        .set("spark.driver.allowMultipleContexts", "true")
        .setMaster("local[*]");

    JavaSparkContext sc = new JavaSparkContext(conf);

    JavaRDD<Tweet> tweets = sc.textFile(pathToFile).map(new Function<String, Tweet>() {
        public Tweet call(String line) throws Exception {
            // TODO Auto-generated method stub
            return Parse.parseJsonToTweet(line);
        }

    });

    return tweets;
  }

  /**
   *  Find all the hashtags mentioned on tweets
   */
  public JavaRDD<String> hashtagMentionedOnTweet() {
    JavaRDD<Tweet> tweets = loadData();

    // You want to return an RDD with the mentions
    // Hint: think about separating the word in the text field and then find the mentions
    // TODO write code here
    JavaRDD<String> mentions = null;

    return mentions;
  }

  /**
   *  Count how many times each hashtag is mentioned
   */
  public JavaPairRDD<String,Integer> countMentions() {
    JavaRDD<String> mentions = hashtagMentionedOnTweet();

    // Hint: think about what you did in the wordcount example
    // TODO write code here
    JavaPairRDD<String, Integer> counts = null;

    return counts;
  }

  /**
   *  Find the 10 most popular Hashtags by descending order
   */
  public List<Tuple2<Integer, String>> top10HashTags() {
    JavaPairRDD<String, Integer> counts = countMentions();

    // Hint: take a look at the sorting and take methods
    // TODO write code here
    List<Tuple2<Integer, String>> top10 = null;

    return top10;
  }

}

InvertedIndex代码块

package tutorial;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;

import utils.Parse;
import utils.Tweet;

import java.util.Map;

/**

 * 目标 ： 建立标记的索引视图
 *
 * 说明：  例如对于标记#spark，它出现在tweet1, tweet3, tweet39中。 建立的索引应该返回(#spark, List(tweet1,tweet3, tweet39))
 *
 */
public class InvertedIndex {

  private static String pathToFile = "data/reduced-tweets.json";

  /**
   *  Load the data from the json file and return an RDD of Tweet
   */
  public JavaRDD<Tweet> loadData() {
    // create spark configuration and spark context
    SparkConf conf = new SparkConf()
        .setAppName("Inverted index")
        .set("spark.driver.allowMultipleContexts", "true")
        .setMaster("local[*]");

    JavaSparkContext sc = new JavaSparkContext(conf);

    JavaRDD<Tweet> tweets = sc.textFile(pathToFile).map(new Function<String, Tweet>() {
        public Tweet call(String line) throws Exception {
            // TODO Auto-generated method stub
            return Parse.parseJsonToTweet(line);
        }

    });

    return tweets;
  }

  public Map<String, Iterable<Tweet>> invertedIndex() {
    JavaRDD<Tweet> tweets = loadData();

    // for each tweet, extract all the hashtag and then create couples (hashtag,tweet)
    // Hint: see the flatMapToPair method
    // TODO write code here
    JavaPairRDD<String, Tweet> pairs = null;

    // We want to group the tweets by hashtag
    // TODO write code here
    JavaPairRDD<String, Iterable<Tweet>> tweetsByHashtag = null;

    // Then return the inverted index (= a map structure)
    // TODO write code here
    Map<String, Iterable<Tweet>> map = null;

    return map;
  }

}