公司数据库有近三千台服务器，用saltstack管理近三千台服务器，会出现各种问题，监控minion端状态非常重要，因为这种监控不属于非常紧急，出了问题就必须立刻处理的故障，所以只需要写好定时任务，定时执行就可以了

代码逻辑：

1、首先从数据库里面查出所有数据库平台的机器
2、通过通道机api调用，在salt master执行一条命令uptime
3、对返回的结果进行分析，如果存在load关键字，则表明该minion的salt状态良好，反之如果没有关键字，则重启salt minion，再检测一次，如果还没有返回，则表示该minion状态为bad，入库

表：

show create table salt_mon \G;
*************************** 1. row ***************************
       Table: salt_mon
Create Table: CREATE TABLE `salt_mon` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
  `ip_in` varchar(16) NOT NULL DEFAULT '',
  `salt_mon_value` varchar(10) NOT NULL DEFAULT '',
  `salt_mon_info` varchar(100) NOT NULL DEFAULT '',
  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  PRIMARY KEY (`id`),
  KEY `salt_mon_value` (`salt_mon_value`),
  KEY `ip_in` (`ip_in`),
  KEY `ctime` (`ctime`)
) ENGINE=InnoDB AUTO_INCREMENT=4244 DEFAULT CHARSET=utf8
1 row in set (0.01 sec)

程序代码：

#!/usr/bin/python26
# -*- coding: utf-8 -*-
"""
@author:mujibin
@modify time: 2015-11-05
"""
import os
import sys
import time
import datetime
import re
import paramiko
import socket
os.chdir(sys.path[0])
sys.path.append("/data1/salt/mysqlapi/salt/")
import urllib
import urllib2
import traceback
import json
import logging
import types
import commands
import MySQLdb
from multiprocessing import Pool
#from salt_repair import *
reload(sys)
sys.setdefaultencoding('utf8')
c_date = time.strftime("%Y%m%d",time.localtime())
c_time = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime()) # 统一入库时间，很重要，是前端
最新信息的依据
H3306='**com.cn'
H3308='***com.cn'
P3306=3306
P3308=3308
dp_admin='dp_admin'
HOST_PORT='3306'
HOST_USER = 'mysqlha'
HOST_PASSED = '*******'
db='test'def getIp(domain):import socketmyaddr = socket.getaddrinfo(domain,'http')[0][4][0]return myaddr
MASTERDNS= "********.com.cn"
MASTERIP = getIp(MASTERDNS) ###获取salt-master的ip地址def sql_select(sql, port=3306, domain='*****', db='salt'):##查询port = int(port)array = []try:db = MySQLdb.connect(host=domain,user=HOST_USER,port=port,passwd=HOST_PASSED,db=db,connect_timeout=3,charset="utf8")cursor = db.cursor()cursor.execute(sql)rows = cursor.fetchall()for row in rows:array.append(row)return arrayexcept Exception,e:#print str(e)return array
def sql_insert(sql, port=3306, domain='****', db='salt'):##插入try:db = MySQLdb.connect(host=domain,user=HOST_USER,port=port,passwd=HOST_PASSED,db=db,connect_timeout=3,charset="utf8")cursor = db.cursor()cursor.execute(sql)db.commit()db.close()except Exception,e:#print str(e)db.rollback()db.close()
def ssh_connect_bak(host):client = paramiko.SSHClient()client.load_system_host_keys()client.set_missing_host_key_policy(paramiko.AutoAddPolicy())privatekeyfile = os.path.expanduser('/root/.ssh/id_rsa')mykey = paramiko.RSAKey.from_private_key_file(privatekeyfile)host=host.strip()client.connect(host,26387,username='root',timeout=2,pkey=mykey)return clientdef ssh_connect(host):##ssh连接client = paramiko.SSHClient()client.load_system_host_keys()client.set_missing_host_key_policy(paramiko.AutoAddPolicy())host=host.strip()client.connect(host,26387,username='root',timeout=2)return client'''通过测试发现通过ssh连接到master端执行命令，一旦服务器数量多了，很容易会卡住退不出来，程序也被卡死，没法继续跑下去，设置超时时间也不生效，具体原因没找到，换了公司通道机来进行连接，没有再出现这个问题，用下面这个run_cmd函数'''def run_cmd(ips,cmd,method,output,ignore_error,timeout):_ips_  = ips_cmd_  =  cmd_method_ = method_output_ = output_ignore_error_ = ignore_error_timeout_ = timeout_user_='litao'_key_='*******'url='https://*****.php'argument={ 'user':_user_,'method':_method_,'output':_output_,'ignore_error':_ignore_error_,'key':_key_,'timeout':_timeout_,'ip':_ips_,'cmd':_cmd_}try:data = urllib.urlencode(argument)response = urllib2.urlopen(url, data)except Exception,e:msg = "Call the api function error!"#logger.debug(msg)#logger.debug(e)#print msg#print epassreturn response.read()
def ssh_cmd(host,cmd):###通过ssh去执行命令try:client = ssh_connect(host)i,o,e = client.exec_command(cmd)res = o.read()return resexcept Exception,e:msg = "The host:%s and cmd:%s execute exception." % (host,cmd)#print msgpass
iplist=['10.75.16.197']
# 获取 node表内所有ip
def get_all_id():##获取所有的ip列表sql = """select distinct ip_in from node \where depid=1 \and ip_in not in \(select node.ip_in from node, node2module where node.id=node2module.nid \and node2module.mname  in ('dbstorage') )  """ids = []rows = sql_select(sql, port=3306, domain='***.com.cn', db='**')if rows:for row in rows:if row[0] not in ids:if row[0] not in iplist:ids.append(row[0])return ids
def salt_minion_mon(host,p_status,s_status,re_info): ###处理salt-master端执行命令后的结果，并写入数据库try:status = ""salt_mon_value = ""salt_mon_info = ""insertSql = ""res_ping = p_statussalt_res = s_statusif salt_res == 1:status = "BAD"salt_mon_value = "BAD"salt_mon_info="%s#NULL#NULL#NULL#NULL#%s " %(host,re_info)insertSql = "insert into salt_mon (ip_in, salt_mon_value, salt_mon_info, ctime)  values('%s','%s','%s','%s')" %(host,salt_mon_value,salt_mon_info,c_time)#print insertSqlsql_insert(insertSql)else:status = "OK"salt_mon_value = "OK"salt_mon_info="%s#NULL#NULL#NULL#NULL#%s " %(host,re_info)insertSql = "insert into salt_mon (ip_in, salt_mon_value, salt_mon_info, ctime) values('%s','%s','%s','%s')" %(host,salt_mon_value,salt_mon_info,c_time)#print insertSqlsql_insert(insertSql)except Exception,e:msg = "salt_minion_mon failed!"#print e
def ping_mon_by_host(host): ##获取ping结果try:ping_cmd = "ping -c 1 -w 2 %s > /dev/null"  % hostret = os.system(ping_cmd)if ret == 0:status = 0msg = "The host %s ping ok" % hostelse:status = 1msg = "The host %s ping failed" % hostresult = {"status":status,"msg":msg}return resultexcept Exception,e:msg = """The host %d: ping_mon_by_host failed!""" % host
def check_salt_minion(host):###检查minion端状态，通过一个简单的命令try:#cmd = "ps -ef|grep salt-minion|grep -v grep"cmd = "salt '%s' -t 7 cmd.run 'uptime'" %host#cmd = "salt '%s' test.ping" %host
#        cmd = "uptime" #ret = ssh_cmd(MASTERIP,cmd)#ret = ssh_cmd(MASTERIP,cmd)ret = run_cmd(host,cmd,method="sync",output="text",ignore_error="true",timeout=5)msg = ""#if ret and 'load' in ret:if ret and 'load' in ret:status = 0#msg = 'The host %s salt-minion is running ok.\n' % hostmsg = 'ok'else :restart_salt_minion(host)time.sleep(3)ret1 = run_cmd(host,cmd,method="sync",output="text",ignore_error="true",timeout=5)if ret1 and 'load' in ret1:status = 0 #msg = 'The host %s salt-minion is running ok.\n' % hostmsg = 'ok'else:status = 1#msg = "The host %s salt-minion is running failed.\n" % hosttry:msg = ret1.split(':')[1].strip()##打印出客户端的详细报错except Exception,e:msg = ret1#msg = 'salt-minion failed'result = {'status':status,'message':msg}return resultexcept Exception,e:#traceback.print_exc()pass
def restart_salt_minion(host):##重启minion端try:cmd = '/etc/init.d/salt-minion restart'#ret = ssh_cmd(host,cmd)ret = run_cmd(host,cmd,method="sync",output="text",ignore_error="true",timeout=5)    if ret != None and "Starting salt-minion daemon" in ret and "OK" in ret:status = 0msg = 'The host %s salt-minion restart successed.\n' % hostelse :status = 1msg = "The host %s salt-minion restart failed.\n" % hostresult = {'status':status,'message':msg}return resultexcept Exception,e:#traceback.print_exc()#print epass
def get_salt_minion_status(host):#print '%s test########################'  % hostping_ret = ping_mon_by_host(host)if ping_ret["status"] == 0:salt_ret = check_salt_minion(host)# 如果salt进程不在尝试重启一次saltsalt_status = salt_ret["status"]salt_info =  salt_ret["message"]salt_minion_mon(host,ping_ret["status"],salt_status,salt_info)else:salt_status = 1salt_info = 'ping failed'#delete_key_by_id(host)#insert_salt_status(host,ping_ret["status"],salt_status) #salt_minion_mon(host,ping_ret["status"],salt_status,salt_info)if __name__ == "__main__":begintime = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))begintime = time.strptime(begintime, '%Y-%m-%d %H:%M:%S')begintime = datetime.datetime(*begintime[:6])all_id=get_all_id()pool = Pool(10)##使用进程池pool.map(get_salt_minion_status,all_id)pool.close()pool.join()endtime = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))endtime = time.strptime(endtime, '%Y-%m-%d %H:%M:%S')endtime = datetime.datetime(*endtime[:6])time_dvalue =  (endtime - begintime).secondsprint '总执行时间：%s sec' % (time_dvalue)print '统计的机器数 %s' % (len(all_id))##获取执行过程的时间

把信息录入数据库，然后写一个前端页面，去定时获取数据库最新的信息就可以了，以ctime时间为准