一.  Python 监控脚本说明

DB 服务器是Redhat 5.5. 系统上的Python 是2.4. 所以这套监控脚本也是基于Python 2.* 版本。 Python 3.0 版本的语法上有一定的出入。

1. 1 几点说明：

（1）Python 严格的区分格式的，所以，格式要对其。

（2）脚本里用到了dns.resolver包，脚本里有下载地址。 安装方法参考dns.resolver包的readme文档。

（3）脚本分成2部分，所有的配置信息，放到了healthcheck_config.py里面。 监控部分代码放在了healthcheck.py里了。 配置文件里设定了警告值，超过这个值就会发送报警信息。

（4）本打算把RMAN 的监控也放到里面的。 但是RMAN 每天只需要运行一次。 而这个监控脚本是10分钟执行一次。 所以RMAN 备份的log 会在单独弄个脚本来处理。

（5）脚本里面调用了一些系统命令。 如ifconfig，这个命令只有root用户能执行，所以这个监控脚本也必须使用root用户来部署。 在调试这个脚本的时候，浪费了很多的时间。

（6）脚本里使用了2种方式来发送警告信息，一个是使用smtp发送到邮箱，另一个是通过短信接口发送到手机。 现在一般公司都有自己的邮件服务器。 所以这些在内网就能直接搞定了。 如果说没有短信接口的话，可以发送到移动的139邮箱，或者qq 邮箱， 这2个邮箱都可以设置短信通知邮件。

（7） 这里把所有要监控的服务器，全部添加到脚本配置文件里。 这样在移植的时候，直接部署过去就可以了。 不需要进行其他的修改。

1.2  Python 的基础知识参考一下链接：

Python 基础语法知识一

http://blog.csdn.net/tianlesoftware/archive/2011/02/11/6179869.aspx

Python 基础语法知识二

http://blog.csdn.net/tianlesoftware/archive/2011/02/12/6181166.aspx

Python 执行 系统命令

http://blog.csdn.net/tianlesoftware/archive/2011/02/17/6192202.aspx

二.  监控脚本

2.1 .  参数文件 healthcheck_config.py

#!/usr/bin/python

# coding=gbk

#################################################

# Created by Tianlesoftware

# 2011/2/23

##################################################

# 注意, 如果需要换行时缩进需要跟原有的代码保持统一

##################################################

#CPU 使用率警戒值

#wa: Time spent waiting for IO. Prior to Linux 2.5.41, shown as zero.

MAX_CPU_USED = 70

MAX_WA = 20

FILTERS = ['/boot', '/dev/shm']

MIN_FREE_MB  = 200

#磁盘空间使用率

MAX_USED_PCT = 80

#邮件服务器地址及发送邮件账户和密码

MAIL_FROM='tianlesoftware@vip.qq.com'

SMTP_SERVER='192.168.1.100'

EMAIL_USER='user'

EMAIL_PASSWD='pwd'

#监控的进程信息，以下进程都将被监控

PROCESS = {

'192.168.1.1': ['LISTENER','ora_pmon_wangou','ora_lgwr_wangou','ora_dbw0_wangou','ora_ckpt_wangou','ora_smon_wangou','ora_arc0_wangou'],

'192.168.1.2': ['LISTENER','ora_pmon_wangou','ora_lgwr_wangou','ora_dbw0_wangou','ora_ckpt_wangou','ora_smon_wangou','ora_arc0_wangou'],

'192.168.1.3': ['LISTENER','ora_pmon_xezf','ora_lgwr_xezf','ora_dbw0_xezf','ora_ckpt_xezf','ora_smon_xezf','ora_arc0_xezf'],

'192.168.1.4': ['LISTENER','ora_pmon_xezf','ora_lgwr_xezf','ora_dbw0_xezf','ora_ckpt_xezf','ora_smon_xezf','ora_arc0_xezf'],

'192.168.1.5': ['LISTENER','ora_pmon_XEZF','ora_lgwr_XEZF','ora_dbw0_XEZF','ora_ckpt_XEZF','ora_smon_XEZF','ora_arc0_XEZF'],

}

#监控的文件信息

FILES = {

'192.168.1.1':  [

{'file':'/home/oracle_app/admin/XEZF/bdump/alert_XEZF.log', 'type':'ora-log'},

{'file':'/u01/backup/logs/rman.log', 'type':'rman-log'}],

'192.168.1.2':  [

{'file':'/u01/scripts/rman_backup.sh.out','type':'rman-log'},

{'file':'/u01/app/oracle/admin/wangou/bdump/alert_wangou.log', 'type':'ora-log'}],

'192.168.1.3':  [{'file':'/u01/app/oracle/admin/wangou/bdump/alert_wangou.log', 'type':'ora-log'}],

'192.168.1.4': [{'file':'/u01/app/oracle/admin/xezf/bdump/alert_xezf.log', 'type':'ora-log'}],

'192.168.1.5': [

{'file':'/u01/backup/logs/rman.log','type':'rman-log'},

{'file':'/u01/app/oracle/admin/xezf/bdump/alert_xezf.log', 'type':'ora-log'}]

}

#默认邮箱和手机是DBA的联系方式

DEFAULT_MAILS   = ['tianlesoftware@vip.qq.com']

DEFAULT_MOBILES = '13888888888'

#对应服务器的邮件接收人

MAILS = {

'192.168.1.1': ['tianlesoftware@vip.qq.com',' tianlesoftware@gmail.com'],

'192.168.1.2': ['tianlesoftware@vip.qq.com'],

'192.168.1.3': ['tianlesoftware@vip.qq.com'],

'192.168.1.4': ['tianlesoftware@vip.qq.com'],

'192.168.1.5': ['tianlesoftware@vip.qq.com']

}

#对应服务器负责人手机号

MOBILES = {

'192.168.1.1': '13688888888,13888888888',

'192.168.1.2': '13688888888,13888888888',

'192.168.1.3': '13688888888,13888888888',

'192.168.1.4': '13688888888,13888888888',

'192.168.1.5': '13688888888,13888888888'

}

2.2.  Python 监控脚本： healthcheck.py

#!/usr/bin/python

# coding=gbk

#################################################

# Created by Tianlesoftware

# 2011/2/23

########################################################

# require dns.resolver

# download url: http://www.dnspython.org/kits/1.6.0/

import os

import time

import datetime

import sys

import smtplib

import dns.resolver

import re

import random

import httplib

import urllib2

import urllib

import hc_config

MAX_CPU_USED = hc_config.MAX_CPU_USED

MAX_WA = hc_config.MAX_WA

FILTERS = hc_config.FILTERS

MIN_FREE_MB  = hc_config.MIN_FREE_MB

MAX_USED_PCT = hc_config.MAX_USED_PCT

PROCESS = hc_config.PROCESS

MAILS = hc_config.DEFAULT_MAILS

MOBILES = hc_config.DEFAULT_MOBILES

httplib.HTTPConnection.debuglevel = 1

def uniqify(seq, idfun=None):

# order preserving

if idfun is None:

def idfun(x): return x

seen = {}

result = []

for item in seq:

marker = idfun(item)

# in old Python versions:

# if seen.has_key(marker)

# but in new ones:

if marker in seen: continue

seen[marker] = 1

result.append(item)

return result

def get_ip():

pipe = os.popen('/sbin/ifconfig')

data = pipe.read().strip()

p = re.compile(r'inet addr:([/./d]+)')

r = p.findall(data)

if len(r) > 0:

return r[0]

ip = get_ip()

IP = ip

if hc_config.MAILS.has_key(ip):

MAILS = MAILS + hc_config.MAILS[ip]

MAILS = uniqify(MAILS)

if hc_config.MOBILES.has_key(ip):

MOBILES = MOBILES + ',' + hc_config.MOBILES[ip]

mobiles = MOBILES.split(',')

mobiles = [x.strip() for x in mobiles]

mobiles = uniqify(mobiles)

if len(mobiles) > 0:

MOBILES = mobiles[0]

if len(mobiles) > 1:

for x in mobiles[1:]:

MOBILES = MOBILES + ',' + x

FILES = []

if hc_config.FILES.has_key(ip):

FILES = hc_config.FILES[ip]

########################################################

def load(fname='warn.dat'):

import pickle

fp=open(fname)

d2=fp.read()

fp.close()

o=pickle.loads(d2)

return o

def save(obj,fname='warn.dat'):

# save to file

import pickle

d2 = pickle.dumps(obj)

fp=open(fname, 'w')

fp.write(d2)

fp.close()

########################################################

# for sms

def sendsms(dt,msg):

if len(msg) > 150:

message = msg[0:149]

else:

message = msg

print 'Send to %s %s' % (dt, msg)

f = { 'msg':message }

m = urllib.urlencode(f)

#这里是短信接口的地方，写上自己的短信接口就可以了

url = 'http://.../%s&%s' % (dt,m)

htm = urlget(url)

print htm

def sendsms_s(msg):

sendsms(MOBILES, msg)

def urlget2(url):

request = urllib2.Request(url)

handle = urllib2.urlopen(request)

htm = handle.read()

return htm

def urlget(url):

try:

request = urllib2.Request(url)

handle = urllib2.urlopen(request)

htm = handle.read()

except urllib2.URLError, err:

errno = err[0][0]

errmsg =  err[0][1]

print errno, 'AAA', errmsg

if errno == -2:

request = urllib2.Request(url)

handle = urllib2.urlopen(request)

htm = handle.read()

else:

htm = str(err)

except:

htm = str(sys.exc_info())

return htm

########################################################

# Send mail ....

SMTP_SERVER=hc_config.SMTP_SERVER

EMAIL_USER=hc_config.EMAIL_USER

EMAIL_PASSWD=hc_config.EMAIL_PASSWD

def sendmail(fromaddr, toaddr, subject, body):

server=smtplib.SMTP(SMTP_SERVER)

server.login(EMAIL_USER,EMAIL_PASSWD)

#server.set_debuglevel(1)

#server.sendmail(fromaddr, toaddrs, msg)

msg = 'From: %s/nTo: %s/nSubject: %s/n/n%s/n' % (fromaddr, toaddr, subject, body)

server.sendmail(fromaddr, toaddr, msg)

server.quit()

def sendmail_s(fromaddr, toaddrs, subject, body):

for toaddr in toaddrs:

try:

sendmail(fromaddr, toaddr, subject, body)

except:

info = str(sys.exc_info())

print info

######################################

#

# Requirement:

# General:

#       FS: free space < 5%，or < 200M

#       CPU: Usage > 70%

#       Log: dmesg / messages

#文件系统               1K-块        已用     可用 已用% 挂载点

#/dev/mapper/VolGroup00-LogVol00   14855176   3665468  10422940  27% /

#/dev/sda1               101086     10979     84888  12% /boot

#tmpfs                  1037848         0   1037848   0% /dev/shm

#/dev/mapper/VolGroup00-Home      51735156  13732272  35374892  28% /home

def get_df_data():

cmd = "df -k | awk '(NF > 1){ printf /"%s//n/", $0 }(NF==1){ printf /"%s /", $0}'"

pipe = os.popen(cmd)

data = pipe.read().strip()

lines = data.split('/n')

mps = []

for line in lines[1:]:

fields = line.split()

mp = {}

mp['mount_point'] = fields[-1].strip()

mp['used'] = int(fields[-2].strip().replace('%',''))

mp['free'] = int(fields[-3].strip())

mp['freeM'] = int(fields[-3].strip())/1024

mps.append(mp)

return mps

#procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu------

#r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st

#2  0      0 692900 380608 803308    0    0     1    11   20    1  0  0 99  1  0

#0  0      0 692900 380608 803308    0    0     0     0 1012   94  0  0 100  0  0

def get_vmstat_data():

pipe = os.popen('vmstat -n 1 3')

data = pipe.read().strip()

lines = data.split('/n')

wa = 0

idle = 0

for line in lines[3:]:

fields = line.split()

wa = wa + int(fields[-2].strip())

idle = idle + int(fields[-3].strip())

wa = wa / len(lines[3:])

idle = idle / len(lines[3:])

used = 100-wa-idle

ret = {'wa':wa, 'idle':idle, 'used':used }

return ret

#print 'wa=%d idle=%d used=%d' % (wa, idle, used)

def file_data(fname):

cmd = 'cat "%s"' % (fname)

pipe = os.popen(cmd)

data = pipe.read().strip()

return data

def get_ps_data():

pipe = os.popen('ps -ef')

data = pipe.read().strip()

return data

mail_from =hc_config.MAIL_FROM

vmstat = get_vmstat_data()

mps = get_df_data()

#########################################

#       FS: free space < 5%，or < 200M

#########################################

body = ''

ip=get_ip()

print ip

for item in  mps:

#print item

mp = item['mount_point']

body = body + str(item) + '/n'

if mp not in FILTERS:

alarm = False

msg = '%s (%s) ' % (ip, mp)

if item['freeM'] < MIN_FREE_MB:

msg = msg + 'freeM(%dM)<%dM ' % (item['freeM'],MIN_FREE_MB)

alarm = True

if item['used'] > MAX_USED_PCT:

msg = '%s (%s) used(%d%%)>%d%%' % (ip,mp,item['used'],MAX_USED_PCT)

alarm = True

if alarm:

sendmail_s(mail_from, MAILS, msg, msg)

sendsms_s(msg);

print msg

# CPU: Usage > 70%

if vmstat['used'] > MAX_CPU_USED:

subject = '%s cpu overload %d%% > %d%% ' % (ip, vmstat['used'], MAX_CPU_USED)

body = 'Host %s cpu overload, vmstat: %s' % (ip, str(vmstat))

sendmail_s(mail_from, MAILS, subject, body)

msg = '%s cpu overload %d%% > %d%% ' % (ip, vmstat['used'], MAX_CPU_USED)

sendsms_s(msg);

if vmstat['wa'] > MAX_WA:

subject = '%s cpu wa overload %d%% > %d%% ' % (ip, vmstat['wa'], MAX_WA)

body = 'Host %s cpu wa overload, vmstat: %s' % (ip, str(vmstat))

sendmail_s(mail_from, MAILS, subject, body)

msg = '%s cpu wa overload %d%% > %d%% ' % (ip, vmstat['wa'], MAX_WA)

sendsms_s(msg);

if PROCESS.has_key(ip):

data = get_ps_data()

process = PROCESS[ip]

for proc in process:

if data.find(proc) == -1:

msg = '%s %s is not running' % (ip, proc)

sendmail_s(mail_from, MAILS, msg, data)

sendsms_s(msg)

def ora_log_handle(data,fname,last_ltime=0):

ltime = last_ltime

p  = re.compile(r'((Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) /d+ /d+:/d+:/d+ /d+)', re.DOTALL)

rr = p.findall(data)

if len(rr) > 0:

start = 0

pos = []

for r in rr:

pp = data.find(r[0],start)

if pp != -1:

pos.append(pp)

llen = len(r[0])

start = pp + llen

fdata = [data[pos[i]:pos[i+1]] for i in range(len(pos)-1)]

fdata.append(data[pos[-1]:])

warns = []

for log_item in fdata:

#print '[%s]' % (log_item)

rr2 = p.findall(log_item)

if len(rr2) > 0:

ts = rr2[0][0]

d2 = time.strptime(ts,'%a %b %d %H:%M:%S %Y')

t = time.mktime(d2)

print t, ltime

if log_item.find('ORA-') != -1 and t > ltime:

warns.append((t, log_item[len(ts):]))

for x in warns:

t = x[0]

line = x[1]

tt = datetime.datetime.fromtimestamp(int(t))

ts = tt.strftime('%m/%d_%H:%M:%S')

msg = '%s %s %s %s' % (ip, fname, ts, line)

sendsms_s(msg)

sendmail_s(mail_from, MAILS, msg, msg)

ltime = int(t)

return ltime

#FILE_HANDLES = { 'rman-log':rman_log_handle, 'ora-log':ora_log_handle }

FILE_HANDLES = {'ora-log':ora_log_handle }

DATA_FILE = 'file.dat'

file_objs = {}

if os.path.exists(DATA_FILE) == True:

file_objs = load(DATA_FILE)

else:

save(file_objs, DATA_FILE)

for file in FILES:

fname = file['file']

ty = file['type']

if os.path.exists(fname) == True and FILE_HANDLES.has_key(ty):

print 'checking ', fname

last_mtime = 0

last_ltime = 0

mtime =  os.stat(fname).st_mtime

if file_objs.has_key(fname):

last_mtime = file_objs[fname]['mtime']

last_ltime = file_objs[fname]['ltime']

if mtime > last_mtime:

print fname

fdata = file_data(fname)

handle = FILE_HANDLES[ty]

ltime = handle(fdata,fname,last_ltime)

file_objs[fname] = {'mtime':mtime, 'ltime':ltime}

save(file_objs, DATA_FILE)

#print data

2.3. 添加到crontab

脚本添加到crontab ，每10分钟执行一次，注意，是root用户：

[root@qs-wg-db1 pymonitor]# crontab -l

*/10 * * * * /u01/scripts/pymonitor/hc.py >>/u01/scripts/pymonitor/hc.log 2>1&

Linux Crontab 定时任务 命令详解

http://blog.csdn.net/tianlesoftware/archive/2011/02/17/6192202.aspx

------------------------------------------------------------------------------

Blog： http://blog.csdn.net/tianlesoftware

网上资源： http://tianlesoftware.download.csdn.net

相关视频：http://blog.csdn.net/tianlesoftware/archive/2009/11/27/4886500.aspx

DBA1 群：62697716(满); DBA2 群：62697977(满)

DBA3 群：62697850   DBA 超级群：63306533;

聊天 群：40132017

--加群需要在备注说明Oracle表空间和数据文件的关系，否则拒绝申请