spark sql读hbase

项目背景

spark sql读hbase据说官网如今在写，但还没稳定，所以我基于hbase-rdd这个项目进行了一个封装，当中会区分是否为2进制，假设是就在配置文件里指定为#b,如long#b,还实用了个公司封装的Byte转其它类型，这个假设别人用须要自己实现一套方案。假设我们完毕这一步，将会得到一个DataFrame，后面就能够registerTmpTable，正常使用了使用hiveContext，是由于有一定的orc文件。我这套方案是兼容hbase和hfile的。比方：

val conf = new SparkConf
implicit val sc = new SparkContext(conf)
implicit val hiveContext = new HiveContext(sc)
HbaseMappingUtil.getHbaseDataFrame(tableName,startRow,stopRow).registerTempTable(tableName)
hiveContext.sql("select * from tableName limit 1").show()

配置文件

配置文件：

hbase {
  mapping {
    table {
      usertriat {
        name = "hb_user_trait_7days"
        columnfamily = "stat"
        columns = ["p_du", "p_counts", "p_period_dist"]
            schemas = ["String", "int","string"]
        nullable = [true,false,true]
      }

      toddtest {
        name = "todd_test"
        columnfamily = "cf1"
        columns = ["name", "age"]
        schemas = ["String", "int"]
        nullable = [true, true]
      }

      user {
        name = "hb_user"
        columnfamily = "user"
        columns = ["modifiedTime", "nickname", "isThirdparty"]
        schemas = ["long#b", "string", "boolean"]
        nullable = [true, true, true]
      }

    }
  }
}

就是须要配置一些比方columnfamily。column，是否为空，一定要配，相当于自定格式的一个配置

核心代码

核心代码：

import scala.language._
import unicredit.spark.hbase._
import net.ceedubs.ficus.Ficus._
import org.apache.spark.sql.types._
import org.apache.spark.SparkContext
import com.typesafe.config.ConfigFactory
import org.apache.hadoop.hbase.client.Scan
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.sql.{DataFrame, Row}
import com.ximalaya.tran.{Bytes, PrimitiveByteTrans, Tran}
import java.lang.{Boolean ⇒ JBoolean, Double ⇒ JDouble, Float ⇒ JFloat, Long ⇒ JLong}

/**
  * Created by todd.chen on 16/3/28.
  * email : todd.chen@ximalaya.com
  */
object HbaseMappingUtil {

  lazy val config = ConfigFactory.load()

  def getHbaseDataFrame(table: String)(implicit @transient hiveContext: HiveContext,
                                       @transient sc: SparkContext): DataFrame = {
    getHbaseDataFrame(table, None, None)
  }

  def getHbaseDataFrame(table: String, startRow: Option[String], endRow: Option[String])
                       (implicit @transient hiveContext: HiveContext,
                        @transient sc: SparkContext): DataFrame = {
    lazy val hbasePrefix = s"hbase.mapping.table.$table"
    implicit val hbaseConfig = HBaseConfig()
    implicit def string2Integer(str: String): Integer = new Integer(str)
    val tableName = config.as[String](s"$hbasePrefix.name")
    val columnFamily = config.as[String](s"$hbasePrefix.columnfamily")
    val _columns = config.as[Set[String]](s"$hbasePrefix.columns")
    val _names = _columns.toSeq
    val _schemas = config.as[Seq[String]](s"$hbasePrefix.schemas")
    val _nullAbles = config.as[Seq[Boolean]](s"$hbasePrefix.nullable")
    implicit val columnsZipSchema: Map[String, Tran[_ <: AnyRef, Array[Byte]]] = schemaUtil(table)
    val columns = Map(columnFamily → _columns)
    val rddSchema = StructType(Seq(StructField("id", StringType, false)) ++ createSchema(_names, _schemas, _nullAbles))
    val scan = if (startRow.isDefined && endRow.isDefined) Some(getScan(startRow.get, endRow.get)) else None
    def row2Row(row: (String, Map[String, Map[String, Array[Byte]]])) = {
      val cf = row._2(columnFamily)
      val values = Seq(row._1) ++ _names.map(name ⇒ {
        val bytesArray = cf.getOrElse(name, null)
        arrayByte2Object(bytesArray, name)
      })
      Row(values: _*)
    }
    val rowRdd = if (scan.isDefined) {
      sc.hbase[Array[Byte]](tableName, columns, scan.get).map(row2Row
      )
    } else {
      sc.hbase[Array[Byte]](tableName, columns).map(row2Row)
    }
    hiveContext.createDataFrame(rowRdd, rddSchema)
  }

  private def createSchema(names: Seq[String], schemas: Seq[String], nullAbles: Seq[Boolean]): Seq[StructField] = {
    (names, schemas, nullAbles).zipped.map {
      case (name, schema, isnull) ⇒ (name, schema, isnull)
    }.map(string2StructField)
  }

  private def string2StructField(nameAndStyle: (String, String, Boolean)): StructField = {
    val (name, schema, nullAble) = nameAndStyle
    schema.toLowerCase match {
      case "string" ⇒ StructField(name, StringType, nullAble)
      case "double" ⇒ StructField(name, DoubleType, nullAble)
      case "int" | "int#b" ⇒ StructField(name, IntegerType, nullAble)
      case "long" | "long#b" ⇒ StructField(name, LongType, nullAble)
      case "boolean" ⇒ StructField(name, BooleanType, nullAble)
      case "float" ⇒ StructField(name, FloatType, nullAble)
      case "timestamp" ⇒ StructField(name, TimestampType, nullAble)
      case "date" ⇒ StructField(name, DateType, nullAble)
    }
  }

  private def arrayByte2Object(arrayBytes: Array[Byte], column: String)
                              (implicit columnsZipTran: Map[String, Tran[_ <: AnyRef, Array[Byte]]]) = {
    val tran = columnsZipTran.get(column).get
    tran.from(arrayBytes)
  }

  private def schemaUtil(tableName: String) = {
    lazy val hbasePrefix = s"hbase.mapping.table.$tableName"
    val _columns = config.as[Seq[String]](s"$hbasePrefix.columns")
    val _schemas = config.as[Seq[String]](s"$hbasePrefix.schemas")
    column2Tran(_columns.zip(_schemas))
  }

  private def column2Tran(columnZipSchema: Seq[(String, String)]) = {
    var columnZipTran = Map.empty[String, Tran[_ <: AnyRef, Array[Byte]]]
    columnZipSchema.foreach { cs ⇒
      val (column, schema) = cs
      columnZipTran += column → schema2Tran(schema)
    }
    columnZipTran
  }

  private def schema2Tran(schema: String): Tran[_ <: AnyRef, Array[Byte]] = {
    schema.toLowerCase match {
      case "string" ⇒ PrimitiveByteTrans.getTran(classOf[String])
      case "boolean" ⇒ PrimitiveByteTrans.getTran(classOf[JBoolean])
      case "double" ⇒ PrimitiveByteTrans.getTran(classOf[JDouble])
      case "float" ⇒ PrimitiveByteTrans.getTran(classOf[JFloat])
      case "long" ⇒ new Tran[JLong, Array[Byte]] {
        override def from(to: Array[Byte]): JLong = {
          val num = Bytes.toString(to)
          if (num == null) null else new JLong(num)
        }

        override def to(from: JLong): Array[Byte] = Bytes.toBytes(from.toString)
      }
      case "long#b" ⇒ PrimitiveByteTrans.getTran(classOf[JLong])
      case "int" ⇒ new Tran[Integer, Array[Byte]] {
        override def from(to: Array[Byte]): Integer = {
          val num = Bytes.toString(to)
          if (num == null) null else new Integer(num)
        }

        override def to(from: Integer): Array[Byte] = Bytes.toBytes(from.toString)
      }
      case "int#b" ⇒ PrimitiveByteTrans.getTran(classOf[java.lang.Integer])
    }
  }

  private def getScan(startRow: String, endRow: String): Scan = {
    val scan = new Scan()
    scan.setStartRow(Bytes.toBytes(startRow))
    scan.setStopRow(Bytes.toBytes(endRow))
    scan
  }
}

mygithub