Spark 读取HBase和SolrCloud数据

Spark1.6.2 读取 HBase 1.2.3

//hbase-common-1.2.3.jar
//hbase-protocol-1.2.3.jar
//hbase-server-1.2.3.jar
//htrace-core-3.1.0-incubating.jar
//metrics-core-2.2.0.jar

  val sparkConf = new SparkConf()
      .setAppName("User")

    // 创建 spark context
    val sc = new SparkContext(sparkConf)
    val sqlContext = new SQLContext(sc)
    import sqlContext.implicits._

    // 创建HBase configuration
    val hBaseConf = HBaseConfiguration.create()
    hBaseConf.set("hbase.zookeeper.quorum", "192.168.1.1,192.168.1.2,192.168.1.3")
    hBaseConf.set("hbase.zookeeper.property.clientPort", ""); // zookeeper端口号
    //设置表名
    hBaseConf.set(TableInputFormat.INPUT_TABLE, "knowledge")

    // 应用newAPIHadoopRDD读取HBase，返回NewHadoopRDD
    val hbaseRDD = sc.newAPIHadoopRDD(hBaseConf,
      classOf[TableInputFormat],
      classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable],
      classOf[org.apache.hadoop.hbase.client.Result])

    // 将数据映射为表  也就是将 RDD转化为 dataframe schema
    // 读取结果集RDD，返回一个MapPartitionsRDD
    val resRDD = hbaseRDD.map(tuple => tuple._2)

    //打印读取数据内容
    val user_knowledge = resRDD.map(r => (Bytes.toString(r.getRow),
      Bytes.toString(r.getValue(Bytes.toBytes("behavior"), Bytes.toBytes("reg_id"))),
      Bytes.toString(r.getValue(Bytes.toBytes("behavior"), Bytes.toBytes("create_user_id"))),
      Bytes.toString(r.getValue(Bytes.toBytes("behavior"), Bytes.toBytes("knowledge_id"))),
      Bytes.toString(r.getValue(Bytes.toBytes("behavior"), Bytes.toBytes("create_time")))) //
      ).toDF("row", "reg_id", "create_user_id", "knowledge_id", "create_time")

    user_knowledge.registerTempTable("user_knowledge")

    // 测试
    val df2 = sqlContext.sql("SELECT * FROM user_knowledge")

    df2.collect.foreach(println)

    sc.stop