CDH集群各组件配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
# 设置代理命令
agent1.sources = r1
agent1.sinks = k1
agent1.channels = c1
# 设置数据源
agent1.sources.r1.type = org.apache.flume.source.kafka.KafkaSource
agent1.sources.r1.batchSize = 1000
agent1.sources.r1.batchDurationMillis = 1000
agent1.sources.r1.kafka.bootstrap.servers = 255.255.255.255:9092
agent1.sources.r1.kafka.consumer.group.id = sync-bigdata
agent1.sources.r1.kafka.topics.regex = ^[a-zA-Z0-9\-]+-sync-bigdata$
agent1.sources.r1.kafka.consumer.request.timeout.ms = 80000
agent1.sources.r1.kafka.consumer.fetch.max.wait.ms=7000
agent1.sources.r1.kafka.consumer.session.timeout.ms = 70000
agent1.sources.r1.kafka.consumer.heartbeat.interval.ms = 60000
agent1.sources.r1.kafka.consumer.enable.auto.commit = false
# 设置拦截器
agent1.sources.r1.interceptors= i1
agent1.sources.r1.interceptors.i1.type=com.zhuizhi.bigdata.flume.FlumeInterceptor$Builder
# 设置通道类型及缓存设置
agent1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
agent1.channels.c1.kafka.bootstrap.servers = 255.255.255.255:9092
agent1.channels.c1.kafka.topic = kafka-channel-hdfs
agent1.channels.c1.kafka.consumer.group.id = kafka-channel
agent1.channels.c1.kafka.consumer.request.timeout.ms = 80000
agent1.channels.c1.kafka.consumer.fetch.max.wait.ms=7000
agent1.channels.c1.kafka.consumer.session.timeout.ms = 70000
agent1.channels.c1.kafka.consumer.heartbeat.interval.ms = 60000
agent1.channels.c1.kafka.consumer.enable.auto.commit = false
# 设置sink目的地
agent1.sinks.k1.type = hdfs
agent1.sinks.k1.hdfs.path = hdfs://cdh-xxx-xxx-hue/user/hive/warehouse/ods_tmp_t.db/o_flume_kafka_data_origin/dt=%{eventDate}
agent1.sinks.k1.hdfs.filePrefix = log_%Y%m%d_%H
agent1.sinks.k1.hdfs.fileType=DataStream
agent1.sinks.k1.hdfs.rollCount = 0
agent1.sinks.k1.hdfs.rollSize = 134217728
agent1.sinks.k1.hdfs.rollInterval = 600
agent1.sinks.k1.hdfs.batchSize = 100
agent1.sinks.k1.hdfs.threadsPoolSize = 10
agent1.sinks.k1.hdfs.idleTimeout = 0
agent1.sinks.k1.hdfs.minBlockReplicas = 1
agent1.sinks.k1.hdfs.useLocalTimeStamp = true
agent1.sinks.k1.hdfs.timeZone = Asia/Shanghai
# 将数据源和目的地绑定到通道上
agent1.sources.r1.channels = c1
agent1.sinks.k1.channel = c1

1
/etc/hive/auxlib

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
<property>
<name>hive.spark.client.future.timeout</name>
<value>1800</value>
<description>Timeout for requests from Hive client to remote Spark driver.</description>
</property>
<property>
<name>hive.spark.client.connect.timeout</name>
<value>30000</value>
<description>Timeout for remote Spark driver in connecting back to Hive client.</description>
</property>
<property>
<name>hive.spark.client.server.connect.timeout</name>
<value>300000</value>
</property>
<property>
<name>hive.txn.manager</name>
<value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
</property>
<property>
<name>hive.compactor.initiator.on</name>
<value>true</value>
</property>
<property>
<name>hive.compactor.worker.threads</name>
<value>1</value>
</property>
<property>
<name>hive.support.concurrency</name>
<value>true</value>
</property>
<property>
<name>hive.enforce.bucketing</name>
<value>true</value>
</property>
<property>
<name>hive.exec.dynamic.partition.mode</name>
<value>nonstrict</value>
<description>支持动态分区</description>
</property>
<property>
<name>hive.exec.dynamic.partition</name>
<value>true</value>
</property>
<property>
<name>hive.warehouse.subdir.inherit.perms</name>
<value>false</value>
</property>
<property>
<name>hive.exec.stagingdir</name>
<value>/tmp/hive/.hive-staging</value>
</property>
<property>
<name>HIVE_AUXLIB_JARS_PATH</name>
<value>/etc/hive/auxlib</value>
<description>hive辅助jar包存放目录</description>
</property>

1
show create database hive ;

1
alter database hive default character set latin1 ;

1
2
3
4
5
6
use hive;
alter table COLUMNS_V2 modify column COMMENT varchar(256) character set utf8;
alter table TABLE_PARAMS modify column PARAM_VALUE varchar(4000) character set utf8;
alter table PARTITION_PARAMS modify column PARAM_VALUE varchar(4000) character set utf8;
alter table PARTITION_KEYS modify column PKEY_COMMENT varchar(4000) character set utf8;
alter table INDEX_PARAMS modify column PARAM_VALUE varchar(4000) character set utf8;