记一次open-falcon手动push数据

nginx状态码监控

#falcon的agent部署目录
[root@nginxtest scripts]# pwd
/home/falcon/scripts
[root@nginxtest scripts]# cat sentry_count.sh
#!/bin/bash
#sentry数量统计
ts=`date +%s`
t=`LC_ALL=en_US.UTF-8 date '+%d/%b/%Y:%H:%M' -d "-1 minute"`
valus=`tail -1000000 /usr/local/nginx/logs/access.log |egrep 'sentry.chuntianu.me' |egrep 'POST'|wc -l`
curl -X POST -d "[{"metric": "sentry_nums", "endpoint": "nginxtest", "timestamp": $ts,"step": 60,"value": $valus,"counterType": "GAUGE","tags": "request=post,nginx_cluster=nginxtest"}]" http://127.0.0.1:1988/v1/push
[root@nginxtest scripts]# cat nginx.sh
#!/bin/bash
#访问状态统计
codes="302 400 401 403 404 500 502 503"
for code in $codes;do
ts=`date +%s`;
t=`LC_ALL=en_US.UTF-8 date '+%d/%b/%Y:%H:%M' -d "-1 minute"`
valus=`tail -1000000 /usr/local/nginx/logs/access.log | grep -v 'tvdoctor' | grep $t | awk '{print $9}' | grep $code | wc -l`
curl -X POST -d "[{"metric": "nginx_code", "endpoint": "nginxtest", "timestamp": $ts,"step": 60,"value": $valus,"counterType": "GAUGE","tags": "code=$code,nginx_cluster=nginxtest"}]" http://127.0.0.1:1988/v1/push

调用数据

在open-falcon的expression配置相关的表达式即可！

该表达式的功能是采集sentry消息的数量，如果数量连续3次大于100则告警！

参考文档

http://book.open-falcon.org/zh/usage/data-push.html
http://book.open-falcon.org/zh/philosophy/tags-and-hostgroup.html
https://github.com/GuyCheung/falcon-ngx_metric

etcd监控

action1

#!/bin/bash
#端口监控
CONN_STATUS=`netstat -lntp |egrep ':2379'|egrep -v '127.0.0.1' |wc -l`
curl -X POST -d "[{"metric":etcd_listen "CONN_STATUS", "endpoint": etcd_clusters, "db", "timestamp": $ts,"step": 60,"value": $CONN_STATUS,"counterType": "GAUGE","tags": "service=etcd,port=2379"}]" http://127.0.0.1:1988/v1/push
curl --cacert /home/etcd/ca/ca.pem --cert /home/etcd/ca/client.pem --key /home/etcd/ca/client-key.pem https://127.0.0.1:2379/metrics 2>/dev/null |grep -v '#' >/tmp/etcd_monitor.txt
etcd_items="/tmp/etcd_monitor.txt"
ts=`date +%s`;
#"领导者是否存在; 领导者变化的数量; 共识提案的总数; 共识提案的总数; 目前正在处理的提案数量; 失败提案总数"
items1="has_leader leader_changes_seen_total proposals_applied_total proposals_committed_total proposals_failed_total proposals_pending"
#"wal调用的fsync总延迟时间; 后端调用总延迟时间"
items2="wal_fsync_duration_seconds_count backend_commit_duration_seconds_count"
#"发送给具有ID的对等体的总字节数; 从具有ID的对等方收到的总字节数; 发送给grpc客户端的总字节数; 接收到grpc客户端的总字节数; 同级之间总的往返时间"
items3="peer_sent_bytes_total peer_received_bytes_total grpc_received_bytes_total grpc_sent_bytes_total peer_round_trip_time_seconds_count"
#"打开文件描述符的数量; 允许打开文件描述符的最大数量"
items4="process_open_fds process_max_fds"
all_items="items1 items2 items3 items4"
for item in $all_items;do
#sever监控
if [ $item == items1 ];then
for i in $items1;do
valus=`cat $etcd_items |grep "$i" |awk '{print $2}'`
curl -X POST -d "[{"metric":etcd_server, "etcd_status", "endpoint": etcd_cluster, "timestamp": $ts,"step": 60,"value": $valus,"counterType": "GAUGE","tags": "item=etcd_server_$i,service=etcd"}]" http://127.0.0.1:1988/v1/push
done
#磁盘
elif [ $item == items2 ];then
for i in $items2;do
valus=`cat $etcd_items |grep "$i" |awk '{print $2}'`
curl -X POST -d "[{"metric":etcd_disk, "etcd_status", "endpoint": etcd_cluster, "timestamp": $ts,"step": 60,"value": $valus,"counterType": "GAUGE","tags": "item=etcd_server_$i,service=etcd"}]" http://127.0.0.1:1988/v1/push
done
#网络
elif [ $item == items3 ];then
for i in $items3;do
valus=`cat $etcd_items |grep "$i" |awk '{print $2}'`
curl -X POST -d "[{"metric":etcd_net, "etcd_status", "endpoint": etcd_cluster, "timestamp": $ts,"step": 60,"value": $valus,"counterType": "GAUGE","tags": "item=etcd_server_$i,service=etcd"}]" http://127.0.0.1:1988/v1/push
done
#其他
elif [ $item == items4 ];then
for i in $items4;do
valus=`cat $etcd_items |grep "$i" |awk '{print $2}'`
curl -X POST -d "[{"metric":etcd_fd, "etcd_status", "endpoint": etcd_cluster, "timestamp": $ts,"step": 60,"value": $valus,"counterType": "GAUGE","tags": "item=etcd_server_$i,service=etcd"}]" http://127.0.0.1:1988/v1/push
done
fi
done

采集某时间段内nginx日志

#!/bin/bash
#sentry数量统计
apis="3 24"
for api in $apis;do
ts=`date +%s`;
t1=`LC_ALL=en_US.UTF-8 date '+%d/%b/%Y:%H:%M' -d "-3 minute"`
t2=`LC_ALL=en_US.UTF-8 date '+%d/%b/%Y:%H:%M' -d "-1 minute"`
if [ $api -eq 3 ];then
project=medweb_online
elif [ $api -eq 24 ];then
project=op
fi
valus=`tail -1000000 /usr/local/nginx/logs/access.log |egrep 'sentry.chunyu.me' |sed -n "/$t1/,/$t2/"p |egrep 'POST'|egrep "/api/$api/" |wc -l`
curl -X POST -d "[{"metric": "sentry", "endpoint": "nginxtest", "timestamp": $ts,"step": 60,"value": $valus,"type":"sentry","counterType": "GAUGE","tags": "project=$project"}]" http://127.0.0.1:1988/v1/push
#脚本测试
echo $t
echo $t1
echo $t2
echo $api:$project:$valus
done

etcd监控

#!/bin/bash
monit_keys=`curl --cacert /home/etcd/ca/ca.pem --cert /home/etcd/ca/client.pem --key /home/etcd/ca/client-key.pem https://127.0.0.1:2379/metrics 2>/dev/null |grep -v '#'`
ts=`date +%s`;
#"领导者是否存在; 领导者变化的数量; 共识提案的总数; 共识提案的总数; 目前正在处理的提案数量; 失败提案总数; wal调用的fsync总延迟时间; 后端调用总延迟时间; 发送给具有ID的对等体的总字节数; 发送给grpc客户端的总
字节数; 接收到grpc客户端的总字节数; 同级之间总的往返时间; 打开文件描述符的数量; 允许打开文件描述符的最大数量"
items="etcd_server_has_leader etcd_server_leader_changes_seen_total etcd_server_proposals_applied_total etcd_server_proposals_committed_total etcd_server_proposals_failed_total etcd_server_proposals_pending etcd_disk_wal_fsync_duration_seconds_count etcd_disk_backend_commit_duration_seconds_count etcd_network_peer_sent_bytes_total etcd_network_client_grpc_received_bytes_total etcd_network_client_grpc_sent_bytes_total etcd_network_peer_round_trip_time_seconds_count process_open_fds process_max_fds"
for item in $items;do
valus=`echo $monit_keys|xargs -n2 |grep "$item" |awk '{print $2}' |uniq`
if [ "$item"x = "etcd_network_peer_sent_bytes_total"x ];then
valus=`echo $valus |awk '{print $1}' |awk -F'e+' '{print $1}'`
valus=$(printf "%.f" `echo "$valus*10000000"|bc`)
fi
if [ "$item"x == "etcd_network_peer_received_bytes_total"x ];then
valus=`echo $valus |awk '{print $1}' |awk -F'e+' '{print $1}'`
valus=$(printf "%.f" `echo "$valus*1000000"|bc`)
fi
curl -X POST -d "[{"metric": "$item", "endpoint": "configcenter", "timestamp": $ts,"step": 60,"value": $valus,"counterType": "GAUGE","tags": "etcd_cluster=configcenter,srv=etcd"}]" http://127.0.0.1:1988/v1/push
done

注意：本段代码的重点在“valus=echo $monit_keys|xargs -n2 |grep "$item" |awk '{print $2}' |uniq”中 “uniq” 的使用，实际环境中有部分关键字在“$monit_keys”中会重复出现，如果不加uniq会同时有多个数据被push，这样就会导致curl POST数据是报错“cannot decode body”。当然也如果通过关键字过滤出来的monit_keys完全一致，也可以写成“monit_keys=curl --cacert /home/etcd/ca/ca.pem --cert /home/etcd/ca/client.pem --key /home/etcd/ca/client-key.pem https://127.0.0.1:2379/metrics 2>/dev/null |grep -v '#' |uniq”,但是在本脚本中过滤出来的keys值不完全一致，但是value相同，所以只能加在“valus=...”。