lkl风控.随机森林模型测试代码spark1.6

/**
  * Created by lkl on 2017/10/9.
  */
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.SparkConf
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.SparkContext
import org.apache.spark.mllib.tree.RandomForest
import org.apache.spark.mllib.tree.model.RandomForestModel
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.sql.SQLContext
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator
object uvcy {
  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("test") //setMaster("spark://192.168.0.37:7077")
    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)
    val hc = new HiveContext(sc)
    val data2 = hc.sql("select * from  fin_tec.uvcy2")
//第一个字段为身份证号，第二个字段为是否逾期，字符存在在hive中全部为double型
    val data = data2.map{ row => val arr = new ArrayBuffer[Double]()
        for(i <- 2 until row.size){
          if(row.isNullAt(i)){
            arr += 0.0}
          else if(row.get(i).isInstanceOf[Double])
            arr += row.getDouble(i)
          else if(row.get(i).isInstanceOf[Long])
            arr += row.getLong(i).toDouble
          else if(row.get(i).isInstanceOf[String])
            arr += row.getString(i).toDouble}
        LabeledPoint(row.getDouble(1), Vectors.dense(arr.toArray))}
    val splits = data.randomSplit(Array(0.7, 0.3))
    val (trainingData, testData) = (splits(0), splits(1))
    val numClasses = 2
    val categoricalFeaturesInfo = Map[Int, Int]()
    val numTrees = 3
    val featureSubsetStrategy = "auto"
    val impurity = "gini"
    val maxDepth = 4
    val maxBins = 32
    val model = RandomForest.trainClassifier(trainingData, numClasses, categoricalFeaturesInfo, numTrees, featureSubsetStrategy, impurity, maxDepth, maxBins)
    val labelAndPreds = testData.map { point =>
      val prediction = model.predict(point.features)
      (point.label, prediction)
    }
    val evaluator = new MulticlassClassificationEvaluator().setLabelCol("indexedLabel").setPredictionCol("prediction").setMetricName("precision")
    val testErr = labelAndPreds.filter(r => r._1 != r._2).count.toDouble / testData.count()
    println("Test Error = " + testErr)
    println("Learned classification forest model:\n" + model.toDebugString)
    model.save(sc, "uvcymodel/forest")

    val sameModel = RandomForestModel.load(sc, "uvcymodel/forest")
    val data3 = hc.sql("select * from test.uvcy where i_l3_hk_amt=2150")
    val id="110101000000000000"
    val datas = data3.map{ row => val arr = new ArrayBuffer[Double]()
      for(i <- 2 until row.size){
        if(row.isNullAt(i)){
          arr += 0.0}
        else if(row.get(i).isInstanceOf[Double])
          arr += row.getDouble(i)
        else if(row.get(i).isInstanceOf[Long])
          arr += row.getLong(i).toDouble
        else if(row.get(i).isInstanceOf[String])
          arr += row.getString(i).toDouble}
      (Vectors.dense(arr.toArray))}
    val labelAndPreds2 = testData.map { point =>
      val prediction =sameModel.predict(point.features)
      (id,point.label, prediction,point.features)
    }
    labelAndPreds2.take(2)

  }
}