spark 连接 mysql 数据库
在所有master和slave上也要在spark/conf/spark-conf.sh里面设置driver的classpath，解决编译找不到driver的问题http://www.iteblog.com/archives/1300
类似这样：
export SPARK_CLASSPATH=$SPARK_CLASSPATH:/iteblog/com/mysql-connector-java-5.1.3
spark-sql driver要到1.4以上
sbt:
name := "hoursAvg"
version := "1.0"
val apacheSpark = "org.apache.spark" %% "spark-core" % "1.2.0"
val apacheSQL = "mysql" % "mysql-connector-java" % "5.1.37"
val apacheSSQL = "org.apache.spark" % "spark-sql_2.10" % "1.4.0"
lazy val commonSettings = Seq(
organization := "com.gws",
version := "0.1.0",
scalaVersion := "2.10.4"
)
lazy val root = (project in file(".")).
settings(commonSettings: _*).
settings(
name := "hoursAvg",
libraryDependencies ++= Seq (
apacheSQL,
apacheSSQL,
apacheSpark.
exclude("com.esotericsoftware.kryo", "kryo") .
exclude("javax.activation", "activation") .
exclude("commons-logging", "commons-logging") .
exclude("commons-collections", "commons-collections") .
exclude("org.eclipse.jetty.orbit", "javax.transaction") .
exclude("org.eclipse.jetty.orbit", "javax.servlet") .
exclude("org.eclipse.jetty.orbit", "javax.mail.glassfish") .
exclude("org.eclipse.jetty.orbit", "javax.activation")
)
)
scala:
import java.sql.{PreparedStatement, Connection, DriverManager}
import java.util.Properties
import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkContext, SparkConf}
import com.mysql.jdbc.Driver
import java.util.Properties
object hoursAvg {
val host = "IP"
val database = "database name"
def main(args: Array): Unit = {
val sparkConf = new SparkConf().setAppName("RDDRelation").setMaster("spark://Master.Hadoop:7077")
val sc = new SparkContext(sparkConf)
val sqlContext = new SQLContext(sc)
Class.forName("com.mysql.jdbc.Driver")
val url = "jdbc:mysql://IP:3306/databasename"
val prop = new Properties()
prop.setProperty("user","user name")
prop.setProperty("password","password")
val jdbcDF = sqlContext.read.jdbc(url,"AccountStats",prop)
jdbcDF.show(50)
}
}
页:
[1]