LVS负载均衡DR模式+keepalived

wyyy721

　　实验拓扑：

　　

　　实验准备：
　　CentOS6.5-x86-64

node1 10.0.0.11    软件：ipvsadm+keepalived    角色：Director1 　　node2 10.0.0.12    软件：ipvsadm+keepalived    角色：Director2 　　node3 10.0.0.13    软件：nginx    角色：Realerver1 　　node4 10.0.0.14    软件：nginx    角色：Realerver2

　　

　　一、在两台Director上安装配置ipvsadm和keepalived

　　1、安装ipvsadm
# yum install ipvsadm -y
# ipvsadm -v
ipvsadm v1.26 2008/5/15 (compiled with popt and IPVS v1.2.1)　　2、安装keepalived
# sh hm-keepalived-install.sh  （keepalived安装脚本）
#!/bin/bash
# Author: hm  Email: mail@huangming.org
kernel=/usr/src/kernels/$(uname -r)
yum install gcc gcc-c++ pcre-devel openssl-devel popt-devel libnl-devel libnfnetlink libnfnetlink-devel -y
[ -e keepalived-1.2.23.tar.gz ] || wget http://www.keepalived.org/software/keepalived-1.2.23.tar.gz
tar -zxf keepalived-1.2.23.tar.gz && cd keepalived-1.2.23 &&
./configure --prefix=/usr/local/keepalived --sysconf=/etc --with-kernel-dir=${kernel}
make && make install
chkconfig --add keepalived && chkconfig keepalived on
ln -s /usr/local/keepalived/sbin/keepalived /usr/sbin/　　

　　三、在两台Realserver上安装配置nginx
# sh hm-nginx-install.sh（nginx安装脚本，只需将nginx源码包放到/usr/local/src目录下，执行即可）
#!/bin/bash
# Author: hm  Email: mail@huangming.org
nginx_s=/usr/local/src/nginx-1.8.1.tar.gz
nginx_v=nginx-1.8.1
for p in gcc gcc-c++ zlib pcre pcre-devel openssl openssl-devel
do
    if ! rpm -qa | grep -q "^$p";then
        yum install -y $p
    fi
done
if [ ! -d /etc/nginx -o ! -e /usr/sbin/nginx ];then
useradd -s /sbin/nologin nginx -M
cd /usr/local/src && tar -zxf ${nginx_s} -C . && cd $nginx_v &&
./configure \
--user=nginx \
--group=nginx \
--prefix=/usr/local/nginx \
--sbin-path=/usr/sbin/nginx \
--conf-path=/etc/nginx/nginx.conf \
--pid-path=/var/run/nginx.pid \
--lock-path=/var/lock/subsys/nginx \
--with-http_stub_status_module \
--with-http_ssl_module \
--with-http_gzip_static_module \
--http-log-path=/var/log/nginx/access.log \
--error-log-path=/var/log/nginx/error.log \
--http-client-body-temp-path=/tmp/nginx/client_body \
--http-proxy-temp-path=/tmp/nginx/proxy \
--http-fastcgi-temp-path=/tmp/nginx/fastcgi \
--with-http_degradation_module \
--with-http_realip_module \
--with-http_addition_module \
--with-http_sub_module \
--with-pcre
make && make install
chown -R nginx:nginx /usr/local/nginx && mkdir /tmp/nginx/client_body -p && mkdir /etc/nginx/vhosts
# nginx.conf
cat > /etc/nginx/nginx.conf  /etc/nginx/vhosts/www.conf  /data/www/html/index.html
/usr/sbin/nginx -t && /usr/sbin/nginx
echo
echo "--------------------------------"
echo "Install ${nginx_v} success"
echo "Test http://$(ifconfig eth0 | awk -F"[ :]+" '/inet addr/{print $4}')"
else
echo "Already installed ${nginx_v}"
fi　　

　　四、在两台Director上配置keepalived
　　1、配置keepalived
# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
    notification_email {
        mail@huangming.org
        741616710@qq.com
    }
    notification_email_from keepalived@localhost
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id LVS_HA
}
vrrp_instance LVS_HA {
    state BACKUP            # 主备服务器均为BACKUP
    interface eth0          # 指定keepalived监听网络的接口
    virtual_router_id 81    # ID标识，master和backup保持一致
    priority 100            # backup为90
    advert_int 2            # master和backup之间检查同步的时间间隔
    nopreempt               # 不抢占模式，仅在master上设置
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.0.0.20/24 dev eth0   # VIP，可以设置多个，每行一个
    }
}
virtual_server 10.0.0.20 80 {      # (LVS配置)
    delay_loop 5                   # (每隔5秒查询realserver状态)
    lb_algo wlc                    # (lvs算法)
    lb_kind DR                     # (lvs负载均衡模式)
    persistence_timeout 0          # (保持会话时间，0表示永久保持)
    persistence_granularity 255.255.255.255  # (持久连接的粒度，默认为4个255，即一个单独的客户端IP分配到一个Realserver上)
    protocol TCP                   # (转发的协议类型，TCP或UDP)
    sorry_server 10.0.0.11 8080    # (备用Realserver节点，当所有的realserver节点失效后，将启用这个节点)
    real_server 10.0.0.13 80 {   # Realserver
        weight 2                 # (调度权重)
        HTTP_GET {               # (健康检测方式)
            url {                # (指定检测的url，可以指定多个)
            path /index.html     # (url详细路径)
            status_code 200
            }
        connect_port 80          # (检查的端口)
        #bindto 10.0.0.20         # (表示通过此地址来发送请求对服务器进行健康检查)
        connect_timeout 3        # (无响应超时时间)
        nb_get_retry 3           # (失败重试次数)
        delay_before_retry 1     # (重试时间间隔)
        }
    }
    real_server 10.0.0.14 80 {
        weight 2                 # (调度权重)
        HTTP_GET {               # (健康检测方式)
            url {                # (指定检测的url，可以指定多个)
            path /index.html     # (url详细路径)
            status_code 200
            }
        connect_port 80          # (检查的端口)
        #bindto 10.0.0.20         # (表示通过此地址来发送请求对服务器进行健康检查)
        connect_timeout 3        # (无响应超时时间)
        nb_get_retry 3           # (失败重试次数)
        delay_before_retry 1     # (重试时间间隔)
        }
}　　2、分别启动keepalived服务
[root@node1 ~]# service keepalived start
[root@node2 ~]# service keepalived start
[root@node1 ~]# ps aux | grep keepalived
root       4008  0.0  0.1  44928  1080 ?        Ss   19:01   0:00 keepalived -D
root       4009  0.1  0.2  47152  2332 ?        S    19:01   0:00 keepalived -D
root       4010  0.1  0.1  47032  1608 ?        S    19:01   0:00 keepalived -D
root       4018  0.0  0.0 103256   864 pts/0    S+   19:03   0:00 grep keepalived　　3、查看LVS状态

# master上
[root@node1 ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.0.0.20:80 wlc persistent 0
  -> 10.0.0.13:80                 Route   2      0          0         
  -> 10.0.0.14:80                 Route   2      0          0
# backup上
[root@node2 ~]# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.0.0.20:80 wlc persistent 0
  -> 10.0.0.13:80                 Route   2      0          0         
  -> 10.0.0.14:80                 Route   2      0          0　　

　　4、查看VIP（VIP默认分配在priority权值高的Director上）
[root@node1 ~]# ip addr
1: lo:  mtu 65536 qdisc noqueue state UNKNOWN
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0:  mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:5a:9d:e0 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.11/24 brd 10.0.0.255 scope global eth0
    inet 10.0.0.20/24 scope global secondary eth0
    inet6 fe80::20c:29ff:fe5a:9de0/64 scope link
       valid_lft forever preferred_lft forever　　

　　五、在两台Realserver上分别在回环接口上配置vip和关闭arp转发

# 查看系统默认arp规则
[root@node3 ~]# cat /proc/sys/net/ipv4/conf/lo/arp_ignore
0
[root@node3 ~]# cat /proc/sys/net/ipv4/conf/lo/arp_announce
0
[root@node3 ~]# cat /proc/sys/net/ipv4/conf/all/arp_ignore
0
[root@node3 ~]# cat /proc/sys/net/ipv4/conf/all/arp_announce
0
在两台Realserver上执行以下脚本
# lvs_dr_rs.sh
#!/bin/bash
vip=10.0.0.20
ifconfig lo:0 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip lo:0
echo "1" > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" > /proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" > /proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" > /proc/sys/net/ipv4/conf/all/arp_announce

[root@node3 ~]# ip a
1: lo:  mtu 16436 qdisc noqueue state UNKNOWN
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet 10.0.0.20/32 brd 10.0.0.20 scope global lo:0
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0:  mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:c5:b4:19 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.13/24 brd 10.0.0.255 scope global eth0
    inet6 fe80::20c:29ff:fec5:b419/64 scope link
       valid_lft forever preferred_lft forever
[root@node4 ~]# ip a
1: lo:  mtu 16436 qdisc noqueue state UNKNOWN
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet 10.0.0.20/32 brd 10.0.0.20 scope global lo:0
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0:  mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:42:34:4f brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.14/24 brd 10.0.0.255 scope global eth0
    inet6 fe80::20c:29ff:fe42:344f/64 scope link
       valid_lft forever preferred_lft forever　　

　　六、测试LVS负载功能
# 这里分别在node3和node4上访问VIP，在我的keepalived配置里我配置了wlc最小连接调度算法，
  和权值为2，所以访问的效果如下
[root@node3 ~]# curl http://10.0.0.20
10.0.0.13
[root@node3 ~]# curl http://10.0.0.20
10.0.0.13
[root@node4 ~]# curl http://10.0.0.20
10.0.0.14
[root@node4 ~]# curl http://10.0.0.20
10.0.0.14　　

　　七、测试主Director故障，看是否实现故障转移（vip转至backup上）

# 关闭主Director的keepalived服务
[root@node1 ~]# service keepalived stop
Stopping keepalived:                                       [  OK  ]
# 查看BACKUP的ip
[root@node2 ~]# ip a
1: lo:  mtu 65536 qdisc noqueue state UNKNOWN
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0:  mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:42:43:20 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.12/24 brd 10.0.0.255 scope global eth0
    inet 10.0.0.20/24 scope global secondary eth0
    inet6 fe80::20c:29ff:fe42:4320/64 scope link
       valid_lft forever preferred_lft forever
      
# 测试是否还能正常通过vip访问web服务
[root@node3 ~]# curl http://10.0.0.20
10.0.0.13
[root@node3 ~]# curl http://10.0.0.20
10.0.0.13
[root@node4 ~]# curl http://10.0.0.20
10.0.0.14
[root@node4 ~]# curl http://10.0.0.20
10.0.0.14　　

　　八、在Realserver配置DR模式启动服务脚本
# cat lvs-dr-rs.service
#!/bin/bash
. /etc/rc.d/init.d/functions
vip=10.0.0.20
RETVAL=$?
case "$1" in
    start)
        echo "Start realserver"
        ifconfig lo:0 $vip broadcast $vip netmask 255.255.255.255 up
        route add -host $vip dev lo:0
        echo "1" > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo "2" > /proc/sys/net/ipv4/conf/lo/arp_announce
        echo "1" > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo "2" > /proc/sys/net/ipv4/conf/all/arp_announce
    ;;
    stop)
        echo "Stop realserver"
        ifconfig lo:0 down
        echo "0" > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo "0" > /proc/sys/net/ipv4/conf/lo/arp_announce
        echo "0" > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo "0" > /proc/sys/net/ipv4/conf/all/arp_announce
    ;;
    *)
        echo Usage: $0 {start|stop}
        exit 1
esac
exit $RETVAL