使用es-hadoop同步hive和es之间数据

1
wget https://artifacts.elastic.co/downloads/elasticsearch-hadoop/elasticsearch-hadoop-7.6.1.zip

1
2
3
unzip elasticsearch-hadoop-7.6.1.zip
hdfs dfs -mkdir /user/test/es_hadoop/
hdfs dfs -put elasticsearch-hadoop-hive-7.6.1.jar /user/test/es_hadoop/

1
add jar hdfs://hadoop.m1:9000/user/test/es_hadoop/elasticsearch-hadoop-hive-7.6.1.jar;

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
CREATE EXTERNAL TABLE elastic_test_table(
   id bigint,
   name string,
   length decimal(4,2),
   birth_day string,
   create_time string
)
STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler'
TBLPROPERTIES(
'es.index.auto.create'='true',--自动创建索引
'es.index.read.missing.as.empty'='true', --防止查询为空值报错
'es.mapping.id'='id', --指定hive的字段对应es索引的_id值
'es.resource'='es_test_table/_doc',  --ES中的索引
'es.net.http.auth.pass'='elastic', --ES密码
'es.net.http.auth.user'='elastic', --ES登录名
'es.nodes'='172.16.227.235,172.16.227.236,172.16.227.229',  --ES地址
'es.port'='9200', 
'es.nodes.wan.only'='true', 
'es.nodes.discovery' = 'false',
'es.read.metadata'='true'
);

1
INSERT overwrite TABLE elastic_test_table VALUES (1, "zhangsan", 20,"birth_day", "create_time");

1
GET /es_test_table/_doc/_search?q=id:<10