spark多数据源hive和mysql联合使用

1
2
spark-2.6.0-bin-2.6.0-cdh5.16.2/jars/

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
public class MoreDatabasesSpark {
    public static void main(String[] args) {
        SparkSession session = SparkSession.builder().appName("SparkSessionApp")
                .master("spark://172.10.70.196:7077")
                .config("hive.metastore.uris", "thrift://172.10.70.196:9083")
                //直接连接hive
                .enableHiveSupport()
                .getOrCreate();
//
        Dataset<Row> peopleData = session.sql("select * from test_db.helloword");
        peopleData.registerTempTable("helloword");
        Dataset<Row> sexData = session.read().format("jdbc").option("url","jdbc:mysql://172.10.70.196:3306/bank_a?createDatabaseIfNotExist=true&useSSL=true&serverTimezone=UTC&useUnicode=true&characterEncoding=UTF-8")
                .option("dbtable", "test_data")
                .option("user", "admin")
                .option("password", "admin")
                .option("driver","com.mysql.cj.jdbc.Driver")
                .load();
        sexData.registerTempTable("sex");
        session.sql("select a.id,a.name,b.memo from helloword a left join sex b on a.id = b.val").show();
        session.stop();
    }
}

1
2
./spark-submit --class org.example.SparkSqlToHive /usr/local/src/test/SparkTest-1.0-SNAPSHOT.jar