flume报错 java.lang.OutOfMemoryError: Java heap space已解决

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
2020-11-26 17:41:25,679 (kafka-coordinator-heartbeat-thread | flume) [ERROR - org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatThread.run(AbstractCoordinator.java:1083)] [Consumer clientId=consumer-2, groupId=flume] Heartbeat thread failed due to unexpected error
java.lang.OutOfMemoryError: Java heap space
at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57)
at java.nio.ByteBuffer.allocate(ByteBuffer.java:335)
at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30)
at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112)
at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335)
at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296)
at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:562)
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:498)
at org.apache.kafka.common.network.Selector.poll(Selector.java:427)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:271)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.pollNoWakeup(ConsumerNetworkClient.java:310)
at org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatThread.run(AbstractCoordinator.java:1020)
2020-11-26 17:41:25,929 (PollableSourceRunner-KafkaSource-r2) [ERROR - org.apache.flume.source.kafka.KafkaSource.doProcess(KafkaSource.java:330)] KafkaSource EXCEPTION, {}
java.lang.RuntimeException: java.lang.OutOfMemoryError: Java heap space
at org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatThread.run(AbstractCoordinator.java:1087)
Caused by: java.lang.OutOfMemoryError: Java heap space
at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57)
at java.nio.ByteBuffer.allocate(ByteBuffer.java:335)
at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30)
at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112)
at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335)
at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296)
at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:562)
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:498)
at org.apache.kafka.common.network.Selector.poll(Selector.java:427)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:271)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.pollNoWakeup(ConsumerNetworkClient.java:310)
at org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatThread.run(AbstractCoordinator.java:1020)
2020-11-26 17:41:26,629 (Thread-11) [INFO - org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient.checkTrustAndSend(SaslDataTransferClient.java:239)] SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
Exception in thread "PollableSourceRunner-KafkaSource-r1" java.lang.OutOfMemoryError: Java heap space
at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57)
at java.nio.ByteBuffer.allocate(ByteBuffer.java:335)
at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30)
at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112)
at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335)
at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296)
at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:562)
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:498)
at org.apache.kafka.common.network.Selector.poll(Selector.java:427)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:271)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:242)
at org.apache.kafka.clients.consumer.KafkaConsumer.pollForFetches(KafkaConsumer.java:1247)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1187)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1154)
at org.apache.flume.source.kafka.KafkaSource.doProcess(KafkaSource.java:216)
at org.apache.flume.source.AbstractPollableSource.process(AbstractPollableSource.java:60)
at org.apache.flume.source.PollableSourceRunner$PollingRunner.run(PollableSourceRunner.java:133)
at java.lang.Thread.run(Thread.java:748)

1
2
3
cd /data/flume/apache-flume/conf/
vim flume-env.sh

1
2
export JAVA_OPTS="-Xms100m -Xmx2000m -Dcom.sun.management.jmxremote"