通过sparkstreaming分析url的数据

spark version 1.6.2

scala verson 2.10.6

此代码参考官方例子----

自定义接收器

import java.io.BufferedReader
 
import org.apache.spark.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.receiver.Receiver
 
import scala.io.{BufferedSource, Source}
 
/**
  * 自定义接收器
  * @param URL url
  */
class UrlReceiver(URL:String) extends Receiver[String](StorageLevel.MEMORY_AND_DISK) with Logging{
  override def onStart(): Unit = {
    new Thread("创建一个线程"){
     override def run(){
       url()
      }
    }.start()
 
  }
 
  override def onStop(): Unit = {
 
  }
 
  private def url(): Unit ={
    var input:String=null
    try{
    val fileContent: BufferedSource = Source.fromURL(URL, "utf-8")
    val reader: BufferedReader = fileContent.bufferedReader()
    input=reader.readLine()
    while (!isStopped && input!=null){
 
        //推送数据给streaming
        store(input)
        input=reader.readLine()
    }
 
    reader.close()
    logInfo("停止接受")
    restart("尝试再次连接~~~~~~~~~~~~~")
    }catch {
      case t:Throwable =>{
        restart(s"接受数据错误${t}")
      }
      case s=>{
        restart(s"连接出现错误${URL}:${s}")
      }
    }
  }
}

写streming程序

import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
 
object urlstreaming {
 
  def main(args: Array[String]): Unit = {
    new SparkConf
   SparkContext
    //此处以百度为例
    val URL= "https://www.baidu.com"
    val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName(s"${this.getClass.getSimpleName}").set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .set("spark.streaming.stopGracefullyOnShutdown", "true")
 
    val ssc = new StreamingContext(conf,Seconds(3))
 
    val value: ReceiverInputDStream[String] = ssc.receiverStream(new UrlReceiver(URL))
 
      value.foreachRDD(
        rdd=>{
 
          rdd.foreach(println)
        }
      )
 
    ssc.start()
    ssc.awaitTermination()
 
  }
}