spark 连接 mysql 数据库

kution

　　在所有master和slave上也要在spark/conf/spark-conf.sh里面设置driver的classpath，解决编译找不到driver的问题
　　http://www.iteblog.com/archives/1300
　　类似这样：
　　export SPARK_CLASSPATH=$SPARK_CLASSPATH:/iteblog/com/mysql-connector-java-5.1.3
　　spark-sql driver要到1.4以上
　　sbt:
　　name := "hoursAvg"
　　version := "1.0"
　　val apacheSpark = "org.apache.spark" %% "spark-core" % "1.2.0"
　　val apacheSQL = "mysql" % "mysql-connector-java" % "5.1.37"
　　val apacheSSQL = "org.apache.spark" % "spark-sql_2.10" % "1.4.0"
　　lazy val commonSettings = Seq(
  organization := "com.gws",
  version := "0.1.0",
  scalaVersion := "2.10.4"
)
　　lazy val root = (project in file(".")).
  settings(commonSettings: _*).
  settings(
    name := "hoursAvg",
    libraryDependencies ++= Seq (
      apacheSQL,
      apacheSSQL,
      apacheSpark.
        exclude("com.esotericsoftware.kryo", "kryo") .
        exclude("javax.activation", "activation") .
        exclude("commons-logging", "commons-logging") .
        exclude("commons-collections", "commons-collections") .
        exclude("org.eclipse.jetty.orbit", "javax.transaction") .
        exclude("org.eclipse.jetty.orbit", "javax.servlet") .
        exclude("org.eclipse.jetty.orbit", "javax.mail.glassfish") .
        exclude("org.eclipse.jetty.orbit", "javax.activation")
    )
  )
　　scala:
　　import java.sql.{PreparedStatement, Connection, DriverManager}
import java.util.Properties
　　import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkContext, SparkConf}
　　import com.mysql.jdbc.Driver
import java.util.Properties
　　object hoursAvg {
　　val host = "IP"
  val database = "database name"
　　def main(args: Array[String]): Unit = {
　　val sparkConf = new SparkConf().setAppName("RDDRelation").setMaster("spark://Master.Hadoop:7077")
      val sc = new SparkContext(sparkConf)
      val sqlContext = new SQLContext(sc)
　　Class.forName("com.mysql.jdbc.Driver")
　　val url = "jdbc:mysql://IP:3306/databasename"
　　val prop = new Properties()
      prop.setProperty("user","user name")
      prop.setProperty("password","password")
　　val jdbcDF = sqlContext.read.jdbc(url,"AccountStats",prop)
　　jdbcDF.show(50)
　　}
}