JVM OOM监控加服务重启 Linux脚本

Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "http-nio2-8801-Acceptor-0"
Exception in thread "SimplePauseDetectorThread_1" java.lang.OutOfMemoryError: GC overhead limit exceeded
java.lang.OutOfMemoryError: GC overhead limit exceeded

Exception in thread "http-nio2-8801-Acceptor-0" java.lang.OutOfMemoryError: Java heap space
    at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57)
02:35:02.282 [localhost-startStop-2-SendThread(iZwz9figezp50iy04h7ufyZ:2181)] WARN  o.a.z.ClientCnxn - [run,1112] - Client session timed out, have not heard from server in 46501ms for sessionid 0x100113ff5af0052
    at java.nio.ByteBuffer.allocate(ByteBuffer.java:335)
    at org.apache.tomcat.util.net.SocketBufferHandler.<init>(SocketBufferHandler.java:41)
    at org.apache.tomcat.util.net.Nio2Endpoint.setSocketOptions(Nio2Endpoint.java:312)
    at org.apache.tomcat.util.net.Nio2Endpoint$Acceptor.run(Nio2Endpoint.java:418)
    at java.lang.Thread.run(Thread.java:748)
02:35:03.959 [localhost-startStop-2-SendThread(iZwz9figezp50iy04h7ufyZ:2181)] INFO  o.a.z.ClientCnxn - [run,1160] - Client session timed out, have not heard from server in 46501ms for sessionid 0x100113ff5af0052, closing socket connection and attempting reconnect

上线时没有做内存快照，根据日志排查也没有发现直接问题。所以先做快照并且加入tomcat监控脚本。
 

通过配置JVM参数做内存快照，--XX:+HeapDumpOnOutOfMemoryError这个参数的含义是：当堆内存溢出时触发输出内存快照。通常搭配--XX:HeapDumpPath参数，指定快照的输出路径；同时可以指定快照文件名，默认快照文件名是 java_<pid>_<date>_<time>_heapdump.hprof。可以通过-XX:OnOutOfMemoryError参数指定发生内存溢出后执行的脚本。

#!/bin/sh

#!/bin/sh

 

# func:系统监控 服务器宕机后，提醒并重启

 

# DEFINE

 

# 定义变量

hostname=`hostname`

nowtime=`date "+%Y-%m-%d-%H:%M:%S"`

host=127.0.0.1:6901

hostip=`hostname -I`

hostdes="$hostname $hostip"

 

# 定义要监控的页面地URL

WebUrl=http://$host/DoNOTremove.txt

 

# 日志输出

GetPageInfo=/tmp/TomcatMonitor.Info

TomcatMonitorLog=/tmp/TomcatMonitor.log

 

# 定义消息的标题

subject="[$hostname]系统运行监控"

subject2="[$hostdes]系统"

body=""

 

function SendMessageToDingding(){

        url="https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

                UA="Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"

                res=`curl -XPOST -s -L -H "Content-Type:application/json" -H "charset:utf-8" $url -d "

                {       "msgtype":"text",

                        "text":{"content":"$1n$2"},

                        "at":{

                        "atMobiles":["18888888888"],

                        "isAtAll":false

                        }

                }"`

        echo $res

}

 

Monitor()

{

echo "$hostdes"

        pidpath=/tmp/tcmonV1.pid      #定义pid文件

                if [ -f "$pidpath" ] ;then

                        echo "[$nowtime]脚本已经在执行,准备退出"

                        cat $pidpath

                exit 2

        else

                echo $$ >$pidpath #将当前Shell进程号记录到pid文件中

                echo "[$nowtime]脚本开始执行"

                cat $pidpath

 

                echo "[$nowtime][info]开始监测linux服务器..."

 

                res=`curl -XPOST -m 10 -s -L -H "Content-Type:application/json" -H "charset:utf-8" $WebUrl `

                echo $res

                if [[ $res =~ "hello running" ]] ;then

                        echo "[$nowtime][success]tomcat运行正常,接口访问正常......"

                else

                        echo "$nowtime][error]访问失败，系统准备自动重启"

                        body="[$nowtime][error]访问失败，系统准备自动重启"

                        SendMessageToDingding $subject $body

 

                        # 定义要重启的脚本路径

                        cd /usr/tomcatTest/bin/

                        ./restartup.sh

                        sleep 5m

 

                        # 检查是否重启成功

                        res=`curl -m 30 -XPOST -s -L -H "Content-Type:application/json" -H "charset:utf-8" $WebUrl `

                        echo $res

                        if [[ $res =~ "hello running" ]] ;then

                                echo "[$nowtime][success]tomcat重启成功,接口访问正常......"

                                body="[$nowtime][success]tomcat重启成功,接口访问正常......"

                                SendMessageToDingding $subject $body

                        else

                                echo "[$nowtime][error]tomcat启动可能失败或者重启超时，请管理员检查"

                                body="[$nowtime][error]tomcat启动可能失败或者重启超时，请管理员检查"

                                SendMessageToDingding $subject $body

                        fi

                fi

                echo "[$nowtime][info]结束监测linux服务器..."

                rm -f $pidpath

        fi

 

}

Monitor>>$TomcatMonitorLog

@ResponseBody
@RequestMapping(value = "testHeap", method = RequestMethod.GET)
public Object testHeap() {

    List<byte[]> list = new ArrayList<>();
    Integer i = 0;
    while(true){
        list.add(new byte[10*1024*1024]);
        System.out.println("内存分配次数：" + (++i));
    }

}