环境说明：

公司是做在线教育的互联网企业，WEB架构为：前端使用LVS + Heartbeat做负载均衡，后端主要是Apache/Nginx + Tomcat，缓存有redis和Memcached,数据库使用的Oracle和Mysql。

脚本实现目的：

通过在服务器本地运行脚本，当检测到服务不可用时，自动重启相关服务，并发邮件通知管理员。

通过脚本检测服务关键配置文件，若文件丢失，则根据备份自动恢复文件，并重启。

脚本思路：

通过脚本访问事先定义好的监控页面，一共访问三次，每次访问间隔几秒钟，若三次访问都失败，则认为服务故障，重启服务并发邮件。配置文件的备份则是通过重启脚本来完成，所有的服务都是通过配置文件的方式传递给脚本，以方便批量部署。

脚本内容：

#!/bin/bash
#This shell-script is use for check tomcat and apache/nginx alive
#Created in 2012-11-01
#Last changed in 2013-05-25
source ~/.bash_profile &>/dev/nulldir=''
dir=/tol/app
dir2=/tol/script
cd $dir2dt2=`date +"%Y-%m-%d"`
ls $dir2/logs &>/dev/null || mkdir -p $dir2/logs &>/dev/null
log="$dir2/logs/tomcatwget-$dt2.log"
host=`/sbin/ifconfig |grep "inet addr"|cut -d ':' -f2 |awk '{print $1}'|head -1`
wgetfile=$dir2/tomcat_wgetfile
conf=$dir2/tomcatwget.conf
conf3=$dir2/web.conf
sh_name=$0function shijian () {dt=`date +"%Y-%m-%d-%H:%M:%S"`
}
function apache_restart () {
#stopuser=`ps  -elf |grep -v grep|grep -v root|grep '/bin/httpd'|grep "$dir"|awk '{print $3}'|head -1`$dir/apache/bin/httpd -k stop  &>/dev/nullsleep 5opid=''opid=`ps -elf |grep -v grep|grep "httpd"|grep "$dir"|awk '{print $4}'`if test -n "$opid" ; thenkill -9  $opidfi
#start/usr/bin/ipcs -s | grep "$user"| gawk '{ print $2 }' | xargs -n 1 ipcrm sem &> /dev/null$dir/apache/bin/httpd -k start  &>/dev/nullnpid=`ps -elf |grep -v grep|grep "httpd"|grep root |grep "$dir"|awk '{print $4}'`shijianif test -z "$npid" ; thenecho "The $dt $host apache restart fail by $sh_name" >> $logecho "The $dt $host apache restart fail by $sh_name" |mail -s "check $host apache restart fail" jiankelseecho "The $dt $host apache is restart,the new pid=$npid" >> $logecho "The $dt $host apache is restart by $sh_name,the new pid=$npid" |mail -s "check $host apache restart" jiankfi
}
function nginx_restart () {
#stopopid=''opid=`cat $dir/nginx/logs/nginx.pid`kill -QUIT $opidsleep 3opid=`ps -elf |grep -v grep|grep process|grep nginx |awk '{print $4}'`if test ! -z "$opid" ; thenkill -9 $opidfi
#start$dir/nginx/sbin/nginx -c $dir/nginx/conf/nginx.confnpid=''npid=`ps -elf |grep -v grep|grep process|grep nginx |awk '{print $4}'`shijianif test -z "$npid" ; thenecho "The $dt $host nginx restart fail by $sh_name" >> $logecho "The $dt $host nginx restart fail by $sh_name" |mail -s "check $host nginx restart fail" jiankelseecho "The $dt $host nginx is restart,the new pid=$npid" >> $logecho "The $dt $host nginx is restart by $sh_name,the new pid=$npid" |mail -s "check $host nginx restart" jiankfi
}
function http_restart () {if test ! -f $conf3 ; thenecho "The $conf3 is not exist" >> $logapache_restartsleep 3nginx_restartelsewhile read linedodir=`echo $line |awk -F\; '{print $1}'`softname=`echo $line |awk -F\; '{print $2}'`$softname\_restartdone < $conf3fi
}
function web_reboot () {http_restartsh $dir2/tomcat-restart.sh $name &>/dev/nullexit 0
}#conf-check
if test ! -f $conf ; thenecho "The $conf is not exist"exit 0
fi
shijian
echo "$dt" >> $log
echo "$host" >> $log##Apache-check
domain=`/usr/bin/head -1 $conf |awk -F\; '{print $1}'`
if test `echo $domain |grep http` ; thenhtml="$domain"
elsehtml=http://$domain/t-s.html
ficat /dev/null > $wgetfile
/usr/bin/elinks -dump  $html >> $wgetfile 2>&1
sleep 3
/usr/bin/elinks -dump  $html >> $wgetfile 2>&1
sleep 3
/usr/bin/elinks -dump  $html >> $wgetfile 2>&1num=''
num=`grep -i -c "ok" $wgetfile`
if (( $num < 1 )) ; thenshijianecho "The $dt $host $html wget error" >> $logecho "The $dt $host $html wget error" | mail -s "check the $host wget error" jiankhttp_restart
elif (( $num < 3&&$num >= 1 )) ; thenecho "The $host $html wget oknumber is $num" |mail -s "$host wget oknumber" jiank
elseecho "The $host $html wget success" >> $log
fiwhile read line
do
shijian
file1=''
file1=`echo $line |awk -F\; '{print $2}'`
file2=''
file2=`echo $line |awk -F\; '{print $3}'`
name=''
name=`echo $line |awk -F\; '{print $4}'`
port=''
port=`echo $line |awk -F\; '{print $5}'`
function pid () {pid=''pid=`ps -elf|grep java|grep "$dir"|grep $name|awk '{print $4}'`
}
function down () {`echo $line |awk -F\; '{print $6}'`sleep 3pidif test -n "$pid" ; thenkill -9 $pidfirm -rf $dir/$name/work/Catalina/*rm -rf $dir/$name/temp/*
}
function up () {`echo $line |awk -F\; '{print $7}'`
}
#web.xml-check
if test "$file1" ; thenif test ! -f $file1 ; thenshijian\cp $file1.bak $file1 && chown tomcat.tomcat $file1 &>/dev/nullif test -f $file1 ; then echo "The $dt $file1 is lost and will be recovery" >> $logecho "The $dt $file1 is lost and will be recovery" | mail -s "check the $host $file1 lost" jiankweb_rebootelseecho "The $dt $file1 is lost and recovery fail" >> $logecho "The $dt $file1 is lost and recovery fail" | mail -s "check the $host $file1 lost" jiankfi elseecho "The $dt $file1 is ok" >> $logfi
fi
#ROOT.xml-check
if test "$file2" ; thenif test ! -f $file2 ; thenshijian\cp $file2.bak $file2 && chown tomcat.tomcat $file2 &>/dev/nullecho "The $dt $file2 do not exist" >> $logecho "The $dt $file2 do not exist" | mail -s "check the $host $file2 lost" jiankdownuppidif test -n "$pid" ; thenecho "$dt $host $file2 is lost and restart by $sh_name" |mail -s "$host $file2 is lost" jiankelseupecho "The $dt $host $file2 is lost and restart fail by $sh_name" >> $logecho "The $dt $host $file2 is lost and restart fail by $sh_name" |mail -s "check $host $name restart fail" jiankfielseecho "The $dt $file2 is ok" >> $logfi
fi
#tomcat-check
if test "$port" ; thenif test `echo $port|grep http` ; thenjsp="$port"elsejsp=http://127.0.0.1:$port/t-s.jspficat /dev/null > $wgetfile
/usr/bin/elinks -dump  $jsp >> $wgetfile 2>&1
sleep 5
/usr/bin/elinks -dump  $jsp >> $wgetfile 2>&1
sleep 5
/usr/bin/elinks -dump  $jsp >> $wgetfile 2>&1num=''
num=`grep -i -c "ok" $wgetfile`if (( $num < 1 )) ; thenshijianecho "The $dt $host $name wget error 3 times and begin to restart" >> $logdownuppidif test -n "$pid" ; thenecho "$dt Restart $name success pid= " $pid >> $logecho "$dt $host $name wget error and restart by $sh_name" |mail -s "$host $name wget error" jiankelseupecho "The $dt $host $name wget error and restart fail by $sh_name" >> $logecho "The $dt $host $name wget error and restart fail by $sh_name" |mail -s "check $host $name restart fail" jiankfi
elif (( $num < 3&&$num >= 1 )) ; thenecho "The $host $name wget oknumber is $num" |mail -s "$host tomcat wget oknumber" jiank
elseecho "The $host $name wget success" >> $log
fifi
done < $conf

脚本配置文件：

127.0.0.1;/tol/htdocs/tomcata/ROOT/WEB-INF/web.xml;/tol/app/tomcata/conf/Catalina/localhost/ROOT.xml;tomcata;7010;/etc/init.d/tomcata stop;/etc/init.d/tomcata start;
;tol/htdocs/tomcatb/ROOT/WEB-INF/web.xml;/tol/app/tomcatb/conf/Catalina/localhost/ROOT.xml;tomcatb;7011;/etc/init.d/tomcatb stop;/etc/init.d/tomcatb start;