Flume "java.lang.NoSuchMethodError: org.apache.hadoop.hbase.client.Put.setWriteToWAL"

113 阅读 0 评论 75 点赞

我是靠谱客的博主陶醉溪流，这篇文章主要介绍Flume "java.lang.NoSuchMethodError: org.apache.hadoop.hbase.client.Put.setWriteToWAL"，现在分享给大家，希望可以做个参考。

之前我们的架构方式采用的是spark+hbase+oozie解析存储及调用算法模式，最近突然出现一个需求，会有很多小文件上传，而且要求达到伪实时处理，也就是秒级别，spark很显然不适合解析了，哪怕是几十行的文件， spark也基本是分钟级别。

我想过2个方案来处理，一个是使用纯JAVA来解析文件，另外一个就是使用flume来解析并直接存储到HBASE。

下载最新版本Flume1.8，通过spoolDir方式，配置文件如下：

复制代码

a1.sources =
r1
a1.sinks =
k1
a1.channels
= c1
a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /data/flume/r1/data
a1.sources.r1.batchSize = 100
a1.sources.r1.channels = c1
a1.channels.c1.type=file
a1.channels.c1.write-timeout=10
a1.channels.c1.keep-alive=10
a1.channels.c1.checkpointDir=/data/flume/c1/checkpoint
a1.channels.c1.dataDirs=/data/flume/c1/data
a1.channels.c1.maxFileSize= 268435456
#a1.sinks.k1.type = logger
a1.sinks.k1.type = hbase
a1.sinks.k1.table = flume
a1.sinks.k1.columnFamily = cf
#a1.sinks.k1.serializer = org.apache.flume.sink.hbase.SimpleAsyncHbaseEventSerializer
a1.sinks.k1.serializer = org.apache.flume.sink.hbase.RegexHbaseEventSerializer
a1.sinks.k1.batchSize = 100
a1.sinks.k1.serializer.regex = (.*?)\|\|(.*?)\|\|(.*?)\|\|(.*?)\|\|(.*)
a1.sinks.k1.serializer.colNames = ROW_KEY,cnc_rdspmeter[0],cnc_rdsvmeter,cnc_statinfo[3],ext_toolno
a1.sinks.k1.serializer.regexIgnoreCase = true
a1.sinks.k1.serializer.depositHeaders = true
a1.sinks.hbaseSink.zookeeperQuorum = datanode01-ucloud.isesol.com:2181
a1.sinks.k1.channel = c1

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
a1.sources =
r1
a1.sinks =
k1
a1.channels
= c1
a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /data/flume/r1/data
a1.sources.r1.batchSize = 100
a1.sources.r1.channels = c1
a1.channels.c1.type=file
a1.channels.c1.write-timeout=10
a1.channels.c1.keep-alive=10
a1.channels.c1.checkpointDir=/data/flume/c1/checkpoint
a1.channels.c1.dataDirs=/data/flume/c1/data
a1.channels.c1.maxFileSize= 268435456
#a1.sinks.k1.type = logger
a1.sinks.k1.type = hbase
a1.sinks.k1.table = flume
a1.sinks.k1.columnFamily = cf
#a1.sinks.k1.serializer = org.apache.flume.sink.hbase.SimpleAsyncHbaseEventSerializer
a1.sinks.k1.serializer = org.apache.flume.sink.hbase.RegexHbaseEventSerializer
a1.sinks.k1.batchSize = 100
a1.sinks.k1.serializer.regex = (.*?)\|\|(.*?)\|\|(.*?)\|\|(.*?)\|\|(.*)
a1.sinks.k1.serializer.colNames = ROW_KEY,cnc_rdspmeter[0],cnc_rdsvmeter,cnc_statinfo[3],ext_toolno
a1.sinks.k1.serializer.regexIgnoreCase = true
a1.sinks.k1.serializer.depositHeaders = true
a1.sinks.hbaseSink.zookeeperQuorum = datanode01-ucloud.isesol.com:2181
a1.sinks.k1.channel = c1

然后启动flume:

复制代码

1
bin/flume-ng agent -n a1 -c conf -f conf/flume-conf.properties

在消费文件的时候错误如下：

复制代码

1
2
3
4
5
6
7
8
9
10
Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor" java.lang.NoSuchMethodError: org.apache.hadoop.hbase.client.Put.setWriteToWAL(Z)Lorg/apache/hadoop/hbase/client/Put;
at org.apache.flume.sink.hbase.HBaseSink$3.run(HBaseSink.java:380)
at org.apache.flume.sink.hbase.HBaseSink$3.run(HBaseSink.java:375)
at org.apache.flume.auth.SimpleAuthenticator.execute(SimpleAuthenticator.java:50)
at org.apache.flume.sink.hbase.HBaseSink.putEventsAndCommit(HBaseSink.java:375)
at org.apache.flume.sink.hbase.HBaseSink.process(HBaseSink.java:345)
at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:67)
at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:145)
at java.lang.Thread.run(Thread.java:748)
^CAttempting to shutdown background worker.

setWriteWal在之前版本存在，但是1.0之后应该就没有了，我不知道为什么Flume的开发者在最新的1.8仍然在使用这个方法，很无奈，查询了一下网上，基本没什么解决方案，于是打开源代码，看看究竟怎么回事。

因为我使用的是type是hbase，因此找到hbaseSink.java, 通过find查找哪里有setWriteWAL, 发现有3个地方存在,

复制代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

public Void run() throws Exception {
for (Row r : actions) {
if (r instanceof Put) {
// ((Put) r).setWriteToWAL(enableWal);
}
// Newer versions of HBase - Increment implements Row.
if (r instanceof Increment) {
//
((Increment) r).setWriteToWAL(enableWal);
}
}
table.batch(actions);
return null;
}

复制代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

public Void run() throws Exception {
List<Increment> processedIncrements;
if (batchIncrements) {
processedIncrements = coalesceIncrements(incs);
} else {
processedIncrements = incs;
}
// Only used for unit testing.
if (debugIncrCallback != null) {
debugIncrCallback.onAfterCoalesce(processedIncrements);
}
for (final Increment i : processedIncrements) {
//
i.setWriteToWAL(enableWal);
table.increment(i);
}
return null;
}
});

上面3个被我注视掉的地方，就是setWriteWAL, 这个东西实际无所谓，因此我很暴力的直接注释，然后再重新打一个包进行替换，官方名字叫：flume-ng-hbase-sink-1.8.0.jar。重新启动Flume,查看结果：

复制代码

hbase(main):001:0> scan 'flume'
ROW
COLUMN+CELL
1529992556110-SzjikLv1LH-0
column=cf:ROW_KEY, timestamp=1529992556407, value=cnc_exeprgname:418
1529992556110-SzjikLv1LH-0
column=cf:cnc_rdspmeter[0], timestamp=1529992556407, value=cnc_rdspmeter[0]:0
1529992556110-SzjikLv1LH-0
column=cf:cnc_rdsvmeter, timestamp=1529992556407, value=cnc_rdsvmeter:6,7,92,0
1529992556110-SzjikLv1LH-0
column=cf:cnc_statinfo[3], timestamp=1529992556407, value=cnc_statinfo[3]:3
1529992556110-SzjikLv1LH-0
column=cf:ext_toolno, timestamp=1529992556407, value=ext_toolno:30
1529992556125-SzjikLv1LH-1
column=cf:ROW_KEY, timestamp=1529992556407, value=cnc_exeprgname:418
1529992556125-SzjikLv1LH-1
column=cf:cnc_rdspmeter[0], timestamp=1529992556407, value=cnc_rdspmeter[0]:0
1529992556125-SzjikLv1LH-1
column=cf:cnc_rdsvmeter, timestamp=1529992556407, value=cnc_rdsvmeter:6,7,93,0
1529992556125-SzjikLv1LH-1
column=cf:cnc_statinfo[3], timestamp=1529992556407, value=cnc_statinfo[3]:3
1529992556125-SzjikLv1LH-1
column=cf:ext_toolno, timestamp=1529992556407, value=ext_toolno:30
1529992556126-SzjikLv1LH-2
column=cf:ROW_KEY, timestamp=1529992556407, value=cnc_exeprgname:418
1529992556126-SzjikLv1LH-2
column=cf:cnc_rdspmeter[0], timestamp=1529992556407, value=cnc_rdspmeter[0]:0
1529992556126-SzjikLv1LH-2
column=cf:cnc_rdsvmeter, timestamp=1529992556407, value=cnc_rdsvmeter:5,10,93,0
1529992556126-SzjikLv1LH-2
column=cf:cnc_statinfo[3], timestamp=1529992556407, value=cnc_statinfo[3]:3
1529992556126-SzjikLv1LH-2
column=cf:ext_toolno, timestamp=1529992556407, value=ext_toolno:30
1529992556127-SzjikLv1LH-3
column=cf:ROW_KEY, timestamp=1529992556407, value=cnc_exeprgname:418
1529992556127-SzjikLv1LH-3
column=cf:cnc_rdspmeter[0], timestamp=1529992556407, value=cnc_rdspmeter[0]:0
1529992556127-SzjikLv1LH-3
column=cf:cnc_rdsvmeter, timestamp=1529992556407, value=cnc_rdsvmeter:7,8,93,0
1529992556127-SzjikLv1LH-3
column=cf:cnc_statinfo[3], timestamp=1529992556407, value=cnc_statinfo[3]:3
1529992556127-SzjikLv1LH-3
column=cf:ext_toolno, timestamp=1529992556407, value=ext_toolno:30
1529992556128-SzjikLv1LH-4
column=cf:ROW_KEY, timestamp=1529992556407, value=cnc_exeprgname:418
1529992556128-SzjikLv1LH-4
column=cf:cnc_rdspmeter[0], timestamp=1529992556407, value=cnc_rdspmeter[0]:0
1529992556128-SzjikLv1LH-4
column=cf:cnc_rdsvmeter, timestamp=1529992556407, value=cnc_r

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
hbase(main):001:0> scan 'flume'
ROW
COLUMN+CELL
1529992556110-SzjikLv1LH-0
column=cf:ROW_KEY, timestamp=1529992556407, value=cnc_exeprgname:418
1529992556110-SzjikLv1LH-0
column=cf:cnc_rdspmeter[0], timestamp=1529992556407, value=cnc_rdspmeter[0]:0
1529992556110-SzjikLv1LH-0
column=cf:cnc_rdsvmeter, timestamp=1529992556407, value=cnc_rdsvmeter:6,7,92,0
1529992556110-SzjikLv1LH-0
column=cf:cnc_statinfo[3], timestamp=1529992556407, value=cnc_statinfo[3]:3
1529992556110-SzjikLv1LH-0
column=cf:ext_toolno, timestamp=1529992556407, value=ext_toolno:30
1529992556125-SzjikLv1LH-1
column=cf:ROW_KEY, timestamp=1529992556407, value=cnc_exeprgname:418
1529992556125-SzjikLv1LH-1
column=cf:cnc_rdspmeter[0], timestamp=1529992556407, value=cnc_rdspmeter[0]:0
1529992556125-SzjikLv1LH-1
column=cf:cnc_rdsvmeter, timestamp=1529992556407, value=cnc_rdsvmeter:6,7,93,0
1529992556125-SzjikLv1LH-1
column=cf:cnc_statinfo[3], timestamp=1529992556407, value=cnc_statinfo[3]:3
1529992556125-SzjikLv1LH-1
column=cf:ext_toolno, timestamp=1529992556407, value=ext_toolno:30
1529992556126-SzjikLv1LH-2
column=cf:ROW_KEY, timestamp=1529992556407, value=cnc_exeprgname:418
1529992556126-SzjikLv1LH-2
column=cf:cnc_rdspmeter[0], timestamp=1529992556407, value=cnc_rdspmeter[0]:0
1529992556126-SzjikLv1LH-2
column=cf:cnc_rdsvmeter, timestamp=1529992556407, value=cnc_rdsvmeter:5,10,93,0
1529992556126-SzjikLv1LH-2
column=cf:cnc_statinfo[3], timestamp=1529992556407, value=cnc_statinfo[3]:3
1529992556126-SzjikLv1LH-2
column=cf:ext_toolno, timestamp=1529992556407, value=ext_toolno:30
1529992556127-SzjikLv1LH-3
column=cf:ROW_KEY, timestamp=1529992556407, value=cnc_exeprgname:418
1529992556127-SzjikLv1LH-3
column=cf:cnc_rdspmeter[0], timestamp=1529992556407, value=cnc_rdspmeter[0]:0
1529992556127-SzjikLv1LH-3
column=cf:cnc_rdsvmeter, timestamp=1529992556407, value=cnc_rdsvmeter:7,8,93,0
1529992556127-SzjikLv1LH-3
column=cf:cnc_statinfo[3], timestamp=1529992556407, value=cnc_statinfo[3]:3
1529992556127-SzjikLv1LH-3
column=cf:ext_toolno, timestamp=1529992556407, value=ext_toolno:30
1529992556128-SzjikLv1LH-4
column=cf:ROW_KEY, timestamp=1529992556407, value=cnc_exeprgname:418
1529992556128-SzjikLv1LH-4
column=cf:cnc_rdspmeter[0], timestamp=1529992556407, value=cnc_rdspmeter[0]:0
1529992556128-SzjikLv1LH-4
column=cf:cnc_rdsvmeter, timestamp=1529992556407, value=cnc_r