###############################################

keepalived

keepalived+lvs实现高可用的负载均衡

测试

###############################################

keepalived

• keepalived一款轻量级高可用软件，工作于layer3, 4 & 5，不同于前几篇博文中的Heartbeat、Corosync等软件的实现机制不同，它采用虚拟路由冗余协议（Virual Router Redundancy Protocal）来实现并且完美的与lvs结合，由于底层使用虚拟路由冗余协议，因此Keepalived具有切换速度快的特点，工作在layer3的keepalived定期向服务器群组中发送ICMP数据包宣告自己存活与否，工作在layer3的keepalived支持以检测TCP端口状态的方式来判定后台Realserver故障与否，自动并将那些判定为故障的后台Realserver从ipvs规则中踢出，工作在layer5可以支持用户自动以脚本来实现相应的智能操作。此lvs也可以结合ldirectord来实现对后台realserver的动态监测，相对于keepalived来说ldirectord属于重量级别的，部署和使用的灵活程度没有前者方便，本文将介绍keepalived。

keepalived+lvs实现高可用的负载均衡

架构图：

realserver端脚本

#!/bin/bash
#
# Script to start LVS DR real server.
# description: LVS DR real server
#
.  /etc/rc.d/init.d/functions
VIP=192.168.1.33
host=`/bin/hostname`
case "$1" in
start)# Start LVS-DR real server on this machine./sbin/ifconfig lo down/sbin/ifconfig lo upecho 1 > /proc/sys/net/ipv4/conf/lo/arp_ignoreecho 2 > /proc/sys/net/ipv4/conf/lo/arp_announceecho 1 > /proc/sys/net/ipv4/conf/all/arp_ignoreecho 2 > /proc/sys/net/ipv4/conf/all/arp_announce/sbin/ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up/sbin/route add -host $VIP dev lo:0
;;
stop)# Stop LVS-DR real server loopback device(s)./sbin/ifconfig lo:0 downecho 0 > /proc/sys/net/ipv4/conf/lo/arp_ignoreecho 0 > /proc/sys/net/ipv4/conf/lo/arp_announceecho 0 > /proc/sys/net/ipv4/conf/all/arp_ignoreecho 0 > /proc/sys/net/ipv4/conf/all/arp_announce
;;
status)# Status of LVS-DR real server.islothere=`/sbin/ifconfig lo:0 | grep $VIP`isrothere=`netstat -rn | grep "lo:0" | grep $VIP`if [ ! "$islothere" -o ! "isrothere" ];then# Either the route or the lo:0 device# not found.echo "LVS-DR real server Stopped."elseecho "LVS-DR real server Running."fi
;;
*)# Invalid entry.echo "$0: Usage: $0 {start|status|stop}"exit 1
;;
esac

安装httpd并建立测试页面如下：

Director端配置

安装ipvsadm和keepalived

yum install ipvsadm
rpm -ivh  keepalived-1.2.7-5.el5.i386.rpm

director_master的配置vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived
global_defs {notification_email {root@localhost   #报警收件人地址}notification_email_from root@localhost  #报警发件人地址smtp_server 127.0.0.1                   #设置smtp服务地址smtp_connect_timeout 30                 #设置连接smtp服务的超时时间router_id LVS_DEVEL                     #发送邮件的主体信息
}
vrrp_script chk_schedown {                 #自定义脚本script "[ -e /etc/keepalived/down ] && exit 1 || exit 0"interval 1    #重试时间间隔weight -5     #减权重fall 2rise 1
}
vrrp_instance VI_1 {state MASTER             #制定keepalived角色interface eth0           #制定检测网络接口virtual_router_id 54     #虚拟路由标示码priority 100             #权重，1-255之间advert_int 1             #设置同步检查的时间间隔，单位是秒authentication {auth_type PASS       #验证类型为PASSauth_pass soulboy    #验证密码}virtual_ipaddress {192.168.1.33/24 dev eth0 label eth0:0  #设置虚拟IP}track_script {chk_schedown}notify_master "/etc/keepalived/notify.sh -n master -a 192.168.1.33"notify_backup "/etc/keepalived/notify.sh -n backup -a 192.168.1.33"notify_fault "/etc/keepalived/notify.sh -n fault -a 192.168.1.33"}
virtual_server 192.168.1.33 80 {  #定义虚拟服务器delay_loop 6                  #设置健康检查时间lb_algo wrr                   #设置负载调度算法lb_kind DR                    #设置LVS工作模式nat_mask 255.255.255.0persistence_timeout 50  protocol TCP                  #设置转发协议的类型sorry_server 127.0.0.1 80     #设置紧急服务器real_server 192.168.1.10 80 {weight 1HTTP_GET {url {path /status_code 200}connect_timeout 2nb_get_retry 3delay_before_retry 1}
}real_server 192.168.1.20 80 {weight 1HTTP_GET {url {path /status_code 200}connect_timeout 2nb_get_retry 3delay_before_retry 1}}}
}

director_backup的配置vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived
global_defs {notification_email {root@localhost}notification_email_from root@localhostsmtp_server 127.0.0.1smtp_connect_timeout 30router_id LVS_DEVEL
}
vrrp_script chk_schedown {script "[ -e /etc/keepalived/down ] && exit 1 || exit 0"interval 1weight -5fall 2rise 1
}
vrrp_instance VI_1 {state BACKUPinterface eth0virtual_router_id 54priority 99advert_int 1authentication {auth_type PASSauth_pass soulboy}virtual_ipaddress {192.168.1.33/24 dev eth0 label eth0:0}track_script {chk_schedown}notify_master "/etc/keepalived/notify.sh -n master -a 192.168.1.33"notify_backup "/etc/keepalived/notify.sh -n backup -a 192.168.1.33"notify_fault "/etc/keepalived/notify.sh -n fault -a 192.168.1.33"
}
virtual_server 192.168.1.33 80 {delay_loop 6lb_algo wrrlb_kind DRnat_mask 255.255.255.0persistence_timeout 50protocol TCPsorry_server 127.0.0.1 80real_server 192.168.1.10 80 {weight 1HTTP_GET {url {path /status_code 200}connect_timeout 2nb_get_retry 3delay_before_retry 1}
}real_server 192.168.1.20 80 {weight 1HTTP_GET {url {path /status_code 200}connect_timeout 2nb_get_retry 3delay_before_retry 1}}}
}

通知脚本vim /etc/keepalived/notify.sh

#!/bin/bash
#
ifalias=${2:-eth0:0}
interface=$(echo $ifalias | awk -F: '{print $1}')
vip=$(ip addr show $interface | grep $ifalias | awk '{print $2}')
contact='root@localhost'
workspace=$(dirname $0)
notify() {subject="$ip change to $1"body="$ip change to $1 $(date '+%F %H:%M:%S')"echo $body | mail -s "$1 transition" $contact
}
case "$1" inmaster)notify masterexit 0;;backup)notify backup/etc/rc.d/init.d/httpd restartexit 0;;fault)notify faultexit 0;;*)echo 'Usage: $(basename $0) {master|backup|fault}'exit 1;;
esac

测试

启动director_master的keepalive服务并查看ipvs规则

#####查看ipvs规则
[root@master ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.33:80 wrr-> 192.168.1.20:80              Route   1      0          0   -> 192.168.1.10:80              Route   1      0          0
#####查看网络信息
[root@master ~]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:C2:5E:01inet addr:192.168.1.61  Bcast:192.168.1.255  Mask:255.255.255.0inet6 addr: fe80::20c:29ff:fec2:5e01/64 Scope:LinkUP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1RX packets:67996 errors:0 dropped:0 overruns:0 frame:0TX packets:116217 errors:0 dropped:0 overruns:0 carrier:0collisions:0 txqueuelen:1000RX bytes:15418633 (14.7 MiB)  TX bytes:8387202 (7.9 MiB)Interrupt:67 Base address:0x2024
eth0:0    Link encap:Ethernet  HWaddr 00:0C:29:C2:5E:01inet addr:192.168.1.33  Bcast:0.0.0.0  Mask:255.255.255.0UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1Interrupt:67 Base address:0x2024

启动director_backup的keepalive服务并查看ipvs规则

#####查看ipvs规则
[root@backup ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.33:80 wrr-> 192.168.1.20:80              Route   1      0          0   -> 192.168.1.10:80              Route   1      0          0
#####查看网络信息
[root@backup ~]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:FA:52:D6inet addr:192.168.1.62  Bcast:192.168.1.255  Mask:255.255.255.0inet6 addr: fe80::20c:29ff:fefa:52d6/64 Scope:LinkUP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1RX packets:115068 errors:0 dropped:0 overruns:0 frame:0TX packets:82940 errors:0 dropped:0 overruns:0 carrier:0collisions:0 txqueuelen:1000RX bytes:19740061 (18.8 MiB)  TX bytes:6476242 (6.1 MiB)Interrupt:67 Base address:0x2024

使用客户端访问VIP

停止director_master的keepalived服务发现VIP消失

[root@master ~]# service keepalived stop
Stopping keepalived:                                       [  OK  ]
[root@master ~]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:C2:5E:01inet addr:192.168.1.61  Bcast:192.168.1.255  Mask:255.255.255.0inet6 addr: fe80::20c:29ff:fec2:5e01/64 Scope:LinkUP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1RX packets:69371 errors:0 dropped:0 overruns:0 frame:0TX packets:118587 errors:0 dropped:0 overruns:0 carrier:0collisions:0 txqueuelen:1000RX bytes:15609985 (14.8 MiB)  TX bytes:8588490 (8.1 MiB)Interrupt:67 Base address:0x2024

在director_backup查看网络信息，发现VIP已成功转移

[root@backup ~]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:FA:52:D6inet addr:192.168.1.62  Bcast:192.168.1.255  Mask:255.255.255.0inet6 addr: fe80::20c:29ff:fefa:52d6/64 Scope:LinkUP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1RX packets:116816 errors:0 dropped:0 overruns:0 frame:0TX packets:84293 errors:0 dropped:0 overruns:0 carrier:0collisions:0 txqueuelen:1000RX bytes:19932196 (19.0 MiB)  TX bytes:6597535 (6.2 MiB)Interrupt:67 Base address:0x2024
eth0:0    Link encap:Ethernet  HWaddr 00:0C:29:FA:52:D6inet addr:192.168.1.33  Bcast:0.0.0.0  Mask:255.255.255.0UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1Interrupt:67 Base address:0x2024

停止realserver_one的httpd服务

[root@realserver_one ~]# service httpd stop
Stopping httpd:                                            [  OK  ]

director_backup查看ipvs规则，发现realserver_one已经被踢出

[root@backup ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.33:80 wrr-> 192.168.1.20:80              Route   1      0          0

客户端访问VIP发现页面恒为node2

停止realserver_two的httpd服务

[root@realserver_two ~]# service httpd stop
Stopping httpd:                                            [  OK  ]

director_backup查看ipvs规则，发现紧急站点生效

[root@backup ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.33:80 wrr-> 127.0.0.1:80                 Local   1      0          0

客户端访问VIP发现页面为自定义警告页面

分别启动realserver_one和realserver_two的httpd服务

#####realserver_one
[root@realserver_one ~]# service httpd start
Starting httpd: httpd: apr_sockaddr_info_get() failed for realserver_one
httpd: Could not reliably determine the server's fully qualified domain name, using 127.0.0.1 for ServerName[  OK  ]
#####realserver_two
[root@realserver_two ~]# service httpd start
Starting httpd: httpd: apr_sockaddr_info_get() failed for realserver_two
httpd: Could not reliably determine the server's fully qualified domain name, using 127.0.0.1 for ServerName[  OK  ]

再次查看director_backup发现ipvs规则已经恢复

[root@backup ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.33:80 wrr-> 192.168.1.20:80              Route   1      0          0   -> 192.168.1.10:80              Route   1      0          0

客户端访问VIP发现负载正常

启动director_master的keepalived服务并查看网络信息发现VIP成功转移

[root@master ~]# service keepalived start
Starting keepalived:                                       [  OK  ]
[root@master ~]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:C2:5E:01inet addr:192.168.1.61  Bcast:192.168.1.255  Mask:255.255.255.0inet6 addr: fe80::20c:29ff:fec2:5e01/64 Scope:LinkUP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1RX packets:70394 errors:0 dropped:0 overruns:0 frame:0TX packets:118644 errors:0 dropped:0 overruns:0 carrier:0collisions:0 txqueuelen:1000RX bytes:15679204 (14.9 MiB)  TX bytes:8593207 (8.1 MiB)Interrupt:67 Base address:0x2024
eth0:0    Link encap:Ethernet  HWaddr 00:0C:29:C2:5E:01inet addr:192.168.1.33  Bcast:0.0.0.0  Mask:255.255.255.0UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1Interrupt:67 Base address:0x2024

在director_backup查看网络信息发现VIP消失

[root@backup ~]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:FA:52:D6inet addr:192.168.1.62  Bcast:192.168.1.255  Mask:255.255.255.0inet6 addr: fe80::20c:29ff:fefa:52d6/64 Scope:LinkUP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1RX packets:118485 errors:0 dropped:0 overruns:0 frame:0TX packets:87004 errors:0 dropped:0 overruns:0 carrier:0collisions:0 txqueuelen:1000RX bytes:20112822 (19.1 MiB)  TX bytes:6791097 (6.4 MiB)Interrupt:67 Base address:0x2024
lo        Link encap:Local Loopbackinet addr:127.0.0.1  Mask:255.0.0.0inet6 addr: ::1/128 Scope:HostUP LOOPBACK RUNNING  MTU:16436  Metric:1RX packets:6781 errors:0 dropped:0 overruns:0 frame:0TX packets:6781 errors:0 dropped:0 overruns:0 carrier:0collisions:0 txqueuelen:0RX bytes:2122280 (2.0 MiB)  TX bytes:2122280 (2.0 MiB)