阿里开源数据同步工具--DataX

46 阅读 0 评论 31 点赞

我是靠谱客的博主健忘香烟，最近开发中收集的这篇文章主要介绍阿里开源数据同步工具--DataX，觉得挺不错的，现在分享给大家，希望可以做个参考。

概述

下载地址：QuickStart
DataX是异构数据源离线同步工具。能够将MySQL sqlServer Oracle Hive HBase FTP 之间进行稳定高效的数据同步。

设计思路：

网状连接-》星型连接

目前支持哪些数据同步？：

核心架构：

推荐使用python 2.6 7 不要使用python 3,0

使用方法和案例：

1. 准备一个job.json 或者job.xml

执行命令：

python datax.py job.json

提供几个json :
1: oracle 到本地文件

{
"job": {
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "xxxxx",
"password": "xxxxxx",
"where": "",
"connection": [
{
"querySql": [
"select * from table1"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.121.30:1521:tcenter"
]
}
]
}
},
"writer": {
"name": "txtfilewriter",
"parameter": {
"path": "D:/data/",
"fileName": "oracledata",
"writeMode": "truncate",
"format": "yyyy-MM-dd"
}
}
}
],
"setting": {
"speed": {
"channel": 1
}
}
}
}

2.oracle ->oracle

{
"job": {
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "xxxx",
"password": "xxxx",
"column": ["*"],
"where": "",
"connection": [
{
"querySql": [
"select * from table1"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.121.30:1521:tcenter"
]
}
]
}
},
"writer": {
       "name": "oraclewriter",
       "parameter": {
"username": "xxxx",
   "password": "xxxx",
   "truncate": "true",
"column": ["SYSTEMNO",
"SYSTEMTIME",
"xxx",
"x",
"x",
"x",
"x",
"x",
"x",
"x",
"x",
"x",
"STATUS",
"x",
"x",
"x"
],
   "connection": [
{

"jdbcUrl": "jdbc:oracle:thin:@192.168.121.30:1521:tcexxnter",
"table": [
"table1"
]
}
]
}
}
}
],
"setting": {
"speed": {
"channel": 6
}
}
}
}

关联查询入库demo:

建表SQL：

CREATE TABLE ZF_PaytDemo(
SystemNo VARCHAR2(32) NOT NULL,
MainOrderSystemNo       VARCHAR2(32)   NOT NULL,
TxAmount NUMBER(16) NOT NULL,
AccountAmount NUMBER(16) NOT NULL,
FinanceChannelCode       VARCHAR2(64)     NOT NULL
);
Json:
{
"job": {
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "xxx",
"password": "xxx",
"where": "",
"connection": [
{
"querySql": [
"select B.SYSTEMNO, A.SYSTEMNO AS MainOrderSystemNo, B.txamount, B.AccountAmount, B.Financechannelcode from zf_fundinordermain A left join zf_fundinorderpayt B on A.Systemno = B.MAINORDERSYSTEMNO where A.Systemno ='35112016091411580603700000' "
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.121.30:1521:tcenter"
]
}
]
}
},
"writer": {
       "name": "oraclewriter",
       "parameter": {
"username": "xxx",
   "password": "xxx",
   "truncate": "false",
"column": [
"SystemNo",
"MainOrderSystemNo",
"TxAmount",
"AccountAmount",
"FinanceChannelCode"
],
   "connection": [
{

"jdbcUrl": "jdbc:oracle:thin:@192.168.121.30:1521:tcenter",
"table": [
"ZF_PaytDemo"
]
}
]
}
}
}
],
"setting": {
"speed": {
"channel": 1
}
}
}
}

file->oracle JSON:

{
"job": {
"content": [
{
"reader": {
"name": "txtfilereader",
"parameter": {
"path": ["D:/datax.txt"],
"encoding": "UTF-8",
"column": [
{
"index": 0,
"type": "string"
},
{
"index": 1,
"type": "string"
},
{
"index": 2,
"type": "long"
},
{
"index": 3,
"type": "long"
},
{
"index": 4,
"type": "string"
}
],
"fieldDelimiter": "t"
}
},
"writer": {
       "name": "oraclewriter",
       "parameter": {
"username": "xxxx",
   "password": "xxxx",
   "truncate": "true",
"column": [
"SystemNo",
"MainOrderSystemNo",
"TxAmount",
"AccountAmount",
"FinanceChannelCode"
],
   "connection": [
{

"jdbcUrl": "jdbc:oracle:thin:@192.168.121.30:1521:tcenter",
"table": [
"ZF_PaytDemo"
]
}
]
}
}
}
],
"setting": {
"speed": {
"channel": 6
}
}
}
}

原文：https://blog.csdn.net/wangming520liwei/article/details/70159537