spark 连接 mysql 数据库

在所有master和slave上也要在spark/conf/spark-conf.sh里面设置driver的classpath，解决编译找不到driver的问题

http://www.iteblog.com/archives/1300

类似这样：

 export SPARK_CLASSPATH=$SPARK_CLASSPATH:/iteblog/com/mysql-connector-java-5.1.3

spark-sql driver要到1.4以上

sbt:

name := "hoursAvg"

version := "1.0"

val apacheSpark = "org.apache.spark" %% "spark-core" % "1.2.0"

val apacheSQL = "mysql" % "mysql-connector-java" % "5.1.37"

val apacheSSQL = "org.apache.spark" % "spark-sql_2.10" % "1.4.0"

lazy val commonSettings = Seq(
organization := "com.gws",
version := "0.1.0",
scalaVersion := "2.10.4"
)

lazy val root = (project in file(".")).
settings(commonSettings: _*).
settings(
name := "hoursAvg",
libraryDependencies ++= Seq (
apacheSQL,
apacheSSQL,
apacheSpark.
exclude("com.esotericsoftware.kryo", "kryo") .
exclude("javax.activation", "activation") .
exclude("commons-logging", "commons-logging") .
exclude("commons-collections", "commons-collections") .
exclude("org.eclipse.jetty.orbit", "javax.transaction") .
exclude("org.eclipse.jetty.orbit", "javax.servlet") .
exclude("org.eclipse.jetty.orbit", "javax.mail.glassfish") .
exclude("org.eclipse.jetty.orbit", "javax.activation")
)
)

scala:

import java.sql.{PreparedStatement, Connection, DriverManager}
import java.util.Properties

import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkContext, SparkConf}

import com.mysql.jdbc.Driver
import java.util.Properties

object hoursAvg {

val host = "IP"
val database = "database name"

def main(args: Array[String]): Unit = {

val sparkConf = new SparkConf().setAppName("RDDRelation").setMaster("spark://Master.Hadoop:7077")
val sc = new SparkContext(sparkConf)
val sqlContext = new SQLContext(sc)

Class.forName("com.mysql.jdbc.Driver")

val url = "jdbc:mysql://IP:3306/databasename"

val prop = new Properties()
prop.setProperty("user","user name")
prop.setProperty("password","password")

val jdbcDF = sqlContext.read.jdbc(url,"AccountStats",prop)

jdbcDF.show(50)

}
}