Spark机器学习3·推荐引擎(spark-shell)

Spark机器学习

准备环境

• jblas
  https://gcc.gnu.org/wiki/GFortranBinaries#MacOS

• org.jblas:jblas:1.2.4-SNAPSHOT

git clone https://github.com/mikiobraun/jblas.git
cd jblas
mvn install

运行环境

cd /Users/erichan/Garden/spark-1.5.1-bin-cdh4

bin/spark-shell --name my_mlib --packages org.jblas:jblas:1.2.4-SNAPSHOT --driver-memory 4G --executor-memory 4G --driver-cores 2

推荐引擎

1 提取有效特征

val PATH = "/Users/erichan/sourcecode/book/Spark机器学习"
val rawData = sc.textFile(PATH+"/ml-100k/u.data")
rawData.first()

res1: String = 196 242 3 881250949

import org.apache.spark.mllib.recommendation.ALS
import org.apache.spark.mllib.recommendation.Rating
val rawRatings=rawData.map(_.split("\t").take(3))
val ratings = rawRatings.map { case Array(user, movie, rating) => Rating(user.toInt, movie.toInt, rating.toDouble) }
ratings.first()

res2: org.apache.spark.mllib.recommendation.Rating = Rating(196,242,3.0)

2 训练推荐模型

val model = ALS.train(ratings, 50, 10, 0.01) //rank=50, iterations=10, lambda=0.01
model.userFeatures.count

res3: Long = 943

model.productFeatures.count

res4: Long = 1682

3 使用模型

3.1.1 用户推荐

val predictedRating = model.predict(789, 123)

val userId = 789
val K = 10
val topKRecs = model.recommendProducts(userId, K)
println(topKRecs.mkString("\n"))

Rating(789,176,5.732688958436494)
Rating(789,201,5.682340265545152)
Rating(789,182,5.5902224300291214)
Rating(789,183,5.5877871075408585)
Rating(789,96,5.4425266495153455)
Rating(789,76,5.39730369058763)
Rating(789,195,5.356822356978749)
Rating(789,589,5.1464233861748925)
Rating(789,134,5.109287533257644)
Rating(789,518,5.106161562126567)

3.1.2 校验推荐

val movies = sc.textFile(PATH+"/ml-100k/u.item")
val titles = movies.map(line => line.split("\\|").take(2)).map(array => (array(0).toInt, array(1))).collectAsMap()
titles(123)
val moviesForUser = ratings.keyBy(_.user).lookup(789)
println(moviesForUser.size)

33

moviesForUser.sortBy(-_.rating).take(10).map(rating => (titles(rating.product), rating.rating)).foreach(println)

(Godfather, The (1972),5.0)
(Trainspotting (1996),5.0)
(Dead Man Walking (1995),5.0)
(Star Wars (1977),5.0)
(Swingers (1996),5.0)
(Leaving Las Vegas (1995),5.0)
(Bound (1996),5.0)
(Fargo (1996),5.0)
(Last Supper, The (1995),5.0)
(Private Parts (1997),4.0)

topKRecs.map(rating => (titles(rating.product), rating.rating)).foreach(println)

(Aliens (1986),5.732688958436494)
(Evil Dead II (1987),5.682340265545152)
(GoodFellas (1990),5.5902224300291214)
(Alien (1979),5.5877871075408585)
(Terminator 2: Judgment Day (1991),5.4425266495153455)
(Carlito's Way (1993),5.39730369058763)
(Terminator, The (1984),5.356822356978749)
(Wild Bunch, The (1969),5.1464233861748925)
(Citizen Kane (1941),5.109287533257644)
(Miller's Crossing (1990),5.106161562126567)

3.2.1 物品推荐

import org.jblas.DoubleMatrix
val aMatrix = new DoubleMatrix(Array(1.0, 2.0, 3.0))
def cosineSimilarity(vec1: DoubleMatrix, vec2: DoubleMatrix): Double = {
    vec1.dot(vec2) / (vec1.norm2() * vec2.norm2())
}
val itemId = 567
val itemFactor = model.productFeatures.lookup(itemId).head
val itemVector = new DoubleMatrix(itemFactor)
cosineSimilarity(itemVector, itemVector)

res10: Double = 1.0

val sims = model.productFeatures.map{ case (id, factor) =>
    val factorVector = new DoubleMatrix(factor)
    val sim = cosineSimilarity(factorVector, itemVector)
    (id, sim)
}
val sortedSims = sims.top(K)(Ordering.by[(Int, Double), Double] { case (id, similarity) => similarity })
println(sortedSims.mkString("\n"))

(567,1.0)
(413,0.7309050775072655)
(895,0.6992030886048359)
(853,0.6960095521899471)
(219,0.6806270119940826)
(302,0.6757242121714326)
(257,0.6721490667554395)
(160,0.6672080746572076)
(563,0.6621573120106216)
(1019,0.6591520069387037)

3.2.2 校验推荐

println(titles(itemId))

Wes Craven's New Nightmare (1994)

val sortedSims2 = sims.top(K + 1)(Ordering.by[(Int, Double), Double] { case (id, similarity) => similarity })
sortedSims2.slice(1, 11).map{ case (id, sim) => (titles(id), sim) }.mkString("\n")

res13: String =
(Tales from the Crypt Presents: Bordello of Blood (1996),0.7309050775072655)
(Scream 2 (1997),0.6992030886048359)
(Braindead (1992),0.6960095521899471)
(Nightmare on Elm Street, A (1984),0.6806270119940826)
(L.A. Confidential (1997),0.6757242121714326)
(Men in Black (1997),0.6721490667554395)
(Glengarry Glen Ross (1992),0.6672080746572076)
(Stephen King's The Langoliers (1995),0.6621573120106216)
(Die xue shuang xiong (Killer, The) (1989),0.6591520069387037)
(Evil Dead II (1987),0.655134288821937)

4 模型效果评估

4.1 均方差（Mean Squared Error，MSE）

val actualRating = moviesForUser.take(1)(0)
val predictedRating = model.predict(789, actualRating.product)
val squaredError = math.pow(predictedRating - actualRating.rating, 2.0)
val usersProducts = ratings.map{ case Rating(user, product, rating)  => (user, product)}
val predictions = model.predict(usersProducts).map{
    case Rating(user, product, rating) => ((user, product), rating)
}
val ratingsAndPredictions = ratings.map{
    case Rating(user, product, rating) => ((user, product), rating)
}.join(predictions)
val MSE = ratingsAndPredictions.map{
    case ((user, product), (actual, predicted)) =>  math.pow((actual - predicted), 2)
}.reduce(_ + _) / ratingsAndPredictions.count
println("Mean Squared Error = " + MSE)

Mean Squared Error = 0.08527363423596633

val RMSE = math.sqrt(MSE)
println("Root Mean Squared Error = " + RMSE)

Root Mean Squared Error = 0.2920164965134099

4.2 K值平均准确率（MAPK）

def avgPrecisionK(actual: Seq[Int], predicted: Seq[Int], k: Int): Double = {
  val predK = predicted.take(k)
  var score = 0.0
  var numHits = 0.0
  for ((p, i) <- predK.zipWithIndex) {
    if (actual.contains(p)) {
      numHits += 1.0
      score += numHits / (i.toDouble + 1.0)
    }
  }
  if (actual.isEmpty) {
    1.0
  } else {
    score / scala.math.min(actual.size, k).toDouble
  }
}
val actualMovies = moviesForUser.map(_.product)
val predictedMovies = topKRecs.map(_.product)
val apk10 = avgPrecisionK(actualMovies, predictedMovies, 10)
val itemFactors = model.productFeatures.map { case (id, factor) => factor }.collect()
val itemMatrix = new DoubleMatrix(itemFactors)
println(itemMatrix.rows, itemMatrix.columns)

(1682,50)

val imBroadcast = sc.broadcast(itemMatrix)
val allRecs = model.userFeatures.map{ case (userId, array) =>
  val userVector = new DoubleMatrix(array)
  val scores = imBroadcast.value.mmul(userVector)
  val sortedWithId = scores.data.zipWithIndex.sortBy(-_._1)
  val recommendedIds = sortedWithId.map(_._2 + 1).toSeq
  (userId, recommendedIds)
}
val userMovies = ratings.map{ case Rating(user, product, rating) => (user, product) }.groupBy(_._1)
val K = 10
val MAPK = allRecs.join(userMovies).map{ case (userId, (predicted, actualWithIds)) =>
  val actual = actualWithIds.map(_._2).toSeq
  avgPrecisionK(actual, predicted, K)
}.reduce(_ + _) / allRecs.count
println("Mean Average Precision at K = " + MAPK)

Mean Average Precision at K = 0.030001472840815356

4.3 MLib内置评估函数·RMSE和MSE

import org.apache.spark.mllib.evaluation.RegressionMetrics
val predictedAndTrue = ratingsAndPredictions.map { case ((user, product), (actual, predicted)) => (actual, predicted) }
val regressionMetrics = new RegressionMetrics(predictedAndTrue)
println("Mean Squared Error = " + regressionMetrics.meanSquaredError)

Mean Squared Error = 0.08527363423596633

println("Root Mean Squared Error = " + regressionMetrics.rootMeanSquaredError)

Root Mean Squared Error = 0.2920164965134099

4.4 MLib内置评估函数·MAP（平均准确率）

import org.apache.spark.mllib.evaluation.RankingMetrics
val predictedAndTrueForRanking = allRecs.join(userMovies).map{ case (userId, (predicted, actualWithIds)) =>
  val actual = actualWithIds.map(_._2)
  (predicted.toArray, actual.toArray)
}
val rankingMetrics = new RankingMetrics(predictedAndTrueForRanking)
println("Mean Average Precision = " + rankingMetrics.meanAveragePrecision)

Mean Average Precision = 0.07208991526855565

val MAPK2000 = allRecs.join(userMovies).map{ case (userId, (predicted, actualWithIds)) =>
  val actual = actualWithIds.map(_._2).toSeq
  avgPrecisionK(actual, predicted, 2000)
}.reduce(_ + _) / allRecs.count
println("Mean Average Precision = " + MAPK2000)

Mean Average Precision = 0.07208991526855561