nginx状态码监控

#falcon的agent部署目录
[root@nginxtest scripts]# pwd
/home/falcon/scripts[root@nginxtest scripts]# cat sentry_count.sh
#!/bin/bash
#sentry数量统计ts=`date +%s`
t=`LC_ALL=en_US.UTF-8 date '+%d/%b/%Y:%H:%M' -d "-1 minute"`
valus=`tail -1000000 /usr/local/nginx/logs/access.log |egrep 'sentry.chuntianu.me' |egrep 'POST'|wc -l`
curl -X POST -d "[{\"metric\": \"sentry_nums\", \"endpoint\": \"nginxtest\", \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"request=post,nginx_cluster=nginxtest\"}]" http://127.0.0.1:1988/v1/push[root@nginxtest scripts]# cat nginx.sh
#!/bin/bash
#访问状态统计
codes="302 400 401 403 404 500 502 503"for code in $codes;dots=`date +%s`;t=`LC_ALL=en_US.UTF-8 date '+%d/%b/%Y:%H:%M' -d "-1 minute"`valus=`tail -1000000 /usr/local/nginx/logs/access.log | grep -v 'tvdoctor' | grep $t | awk '{print $9}' | grep $code | wc -l`curl -X POST -d "[{\"metric\": \"nginx_code\", \"endpoint\": \"nginxtest\", \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"code=$code,nginx_cluster=nginxtest\"}]" http://127.0.0.1:1988/v1/push

调用数据

在open-falcon的expression配置相关的表达式即可！

该表达式的功能是采集sentry消息的数量，如果数量连续3次大于100则告警！

参考文档

http://book.open-falcon.org/zh/usage/data-push.html
http://book.open-falcon.org/zh/philosophy/tags-and-hostgroup.html
https://github.com/GuyCheung/falcon-ngx_metric

etcd监控

action1

#!/bin/bash#端口监控
CONN_STATUS=`netstat -lntp |egrep ':2379'|egrep -v '127.0.0.1' |wc -l`
curl -X POST -d "[{\"metric\":etcd_listen \"CONN_STATUS\", \"endpoint\": etcd_clusters, \"db\", \"timestamp\": $ts,\"step\": 60,\"value\": $CONN_STATUS,\"counterType\": \"GAUGE\",\"tags\": \"service=etcd,port=2379\"}]" http://127.0.0.1:1988/v1/pushcurl --cacert /home/etcd/ca/ca.pem --cert /home/etcd/ca/client.pem --key /home/etcd/ca/client-key.pem https://127.0.0.1:2379/metrics 2>/dev/null |grep -v '#' >/tmp/etcd_monitor.txt
etcd_items="/tmp/etcd_monitor.txt"
ts=`date +%s`;#"领导者是否存在; 领导者变化的数量; 共识提案的总数; 共识提案的总数; 目前正在处理的提案数量; 失败提案总数"
items1="has_leader leader_changes_seen_total proposals_applied_total proposals_committed_total proposals_failed_total proposals_pending"#"wal调用的fsync总延迟时间; 后端调用总延迟时间"
items2="wal_fsync_duration_seconds_count backend_commit_duration_seconds_count"#"发送给具有ID的对等体的总字节数; 从具有ID的对等方收到的总字节数; 发送给grpc客户端的总字节数; 接收到grpc客户端的总字节数; 同级之间总的往返时间"
items3="peer_sent_bytes_total peer_received_bytes_total grpc_received_bytes_total grpc_sent_bytes_total peer_round_trip_time_seconds_count"#"打开文件描述符的数量; 允许打开文件描述符的最大数量"
items4="process_open_fds process_max_fds"all_items="items1 items2 items3 items4"for item in $all_items;do#sever监控if [ $item == items1 ];thenfor i in $items1;dovalus=`cat $etcd_items |grep "$i" |awk '{print $2}'`curl -X POST -d "[{\"metric\":etcd_server, \"etcd_status\", \"endpoint\": etcd_cluster, \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"item=etcd_server_$i,service=etcd\"}]" http://127.0.0.1:1988/v1/pushdone#磁盘elif [ $item == items2 ];thenfor i in $items2;dovalus=`cat $etcd_items |grep "$i" |awk '{print $2}'`curl -X POST -d "[{\"metric\":etcd_disk, \"etcd_status\", \"endpoint\": etcd_cluster, \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"item=etcd_server_$i,service=etcd\"}]" http://127.0.0.1:1988/v1/pushdone#网络elif [ $item == items3 ];thenfor i in $items3;dovalus=`cat $etcd_items |grep "$i" |awk '{print $2}'`curl -X POST -d "[{\"metric\":etcd_net, \"etcd_status\", \"endpoint\": etcd_cluster, \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"item=etcd_server_$i,service=etcd\"}]" http://127.0.0.1:1988/v1/pushdone#其他elif [ $item == items4 ];thenfor i in $items4;dovalus=`cat $etcd_items |grep "$i" |awk '{print $2}'`curl -X POST -d "[{\"metric\":etcd_fd, \"etcd_status\", \"endpoint\": etcd_cluster, \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"item=etcd_server_$i,service=etcd\"}]" http://127.0.0.1:1988/v1/pushdonefi
done

采集某时间段内nginx日志

#!/bin/bash
#sentry数量统计apis="3 24"
for api in $apis;dots=`date +%s`;t1=`LC_ALL=en_US.UTF-8 date '+%d\/%b\/%Y:%H:%M' -d "-3 minute"` t2=`LC_ALL=en_US.UTF-8 date '+%d\/%b\/%Y:%H:%M' -d "-1 minute"` if [ $api -eq 3 ];thenproject=medweb_onlineelif [ $api -eq 24 ];thenproject=opfivalus=`tail -1000000 /usr/local/nginx/logs/access.log |egrep 'sentry.chunyu.me' |sed -n "/$t1/,/$t2/"p |egrep 'POST'|egrep "/api/$api/" |wc -l`curl -X POST -d "[{\"metric\": \"sentry\", \"endpoint\": \"nginxtest\", \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"type\":\"sentry\",\"counterType\": \"GAUGE\",\"tags\": \"project=$project\"}]" http://127.0.0.1:1988/v1/push#脚本测试
echo $t
echo $t1
echo $t2
echo $api:$project:$valus
done

etcd监控

#!/bin/bashmonit_keys=`curl --cacert /home/etcd/ca/ca.pem --cert /home/etcd/ca/client.pem --key /home/etcd/ca/client-key.pem https://127.0.0.1:2379/metrics 2>/dev/null |grep -v '#'`
ts=`date +%s`;
#"领导者是否存在; 领导者变化的数量; 共识提案的总数; 共识提案的总数; 目前正在处理的提案数量; 失败提案总数; wal调用的fsync总延迟时间; 后端调用总延迟时间; 发送给具有ID的对等体的总字节数; 发送给grpc客户端的总
字节数; 接收到grpc客户端的总字节数; 同级之间总的往返时间; 打开文件描述符的数量; 允许打开文件描述符的最大数量"
items="etcd_server_has_leader etcd_server_leader_changes_seen_total etcd_server_proposals_applied_total etcd_server_proposals_committed_total etcd_server_proposals_failed_total etcd_server_proposals_pending etcd_disk_wal_fsync_duration_seconds_count etcd_disk_backend_commit_duration_seconds_count etcd_network_peer_sent_bytes_total etcd_network_client_grpc_received_bytes_total etcd_network_client_grpc_sent_bytes_total etcd_network_peer_round_trip_time_seconds_count process_open_fds process_max_fds"for item in $items;dovalus=`echo $monit_keys|xargs -n2 |grep "$item" |awk '{print $2}' |uniq`if [ "$item"x = "etcd_network_peer_sent_bytes_total"x ];thenvalus=`echo $valus |awk '{print $1}' |awk -F'e+' '{print $1}'`valus=$(printf "%.f" `echo "$valus*10000000"|bc`)fiif [ "$item"x == "etcd_network_peer_received_bytes_total"x ];thenvalus=`echo $valus |awk '{print $1}' |awk -F'e+' '{print $1}'`valus=$(printf "%.f" `echo "$valus*1000000"|bc`)ficurl -X POST -d "[{\"metric\": \"$item\", \"endpoint\": \"configcenter\", \"timestamp\": $ts,\"step\": 60,\"value\": $valus,\"counterType\": \"GAUGE\",\"tags\": \"etcd_cluster=configcenter,srv=etcd\"}]" http://127.0.0.1:1988/v1/pushdone

注意：本段代码的重点在“valus=echo $monit_keys|xargs -n2 |grep "$item" |awk '{print $2}' |uniq”中 “uniq” 的使用，实际环境中有部分关键字在“$monit_keys”中会重复出现，如果不加uniq会同时有多个数据被push，这样就会导致curl POST数据是报错“cannot decode body”。当然也如果通过关键字过滤出来的monit_keys完全一致，也可以写成“monit_keys=curl --cacert /home/etcd/ca/ca.pem --cert /home/etcd/ca/client.pem --key /home/etcd/ca/client-key.pem https://127.0.0.1:2379/metrics 2>/dev/null |grep -v '#' |uniq”,但是在本脚本中过滤出来的keys值不完全一致，但是value相同，所以只能加在“valus=...”。