Keepalived+Nginx实现高可用和双主节点负载均衡

吸毒的虫子

　　简介

　　Nginx可以实现高并发反向代理，lvs集群可以实现负载均衡，但是他们都有一个共同的弊端，就是Nginx，lvs架构中Director是单点故障，有没有一个好的方案解决这个问题呢？答案是有。通过Keepalived就可以实现，前端Nginx，lvs中Director服务器的高可用和负载均衡，通过shell命令或者脚本可以实现对服务器状态和服务的监控！
　　一、环境介绍
　　1、系统环境及软件版本
　　操作系统：Centos6.4-i386
　　软件版本：Nginx-1.4.2
　　keepalived-1.2.7
　　yum源：
　　# vim /etc/yum.repos.d/centos6.repo
[base]
name=centos-base
baseurl=http://mirrors.sohu.com/centos/$releasever/os/$basearch
gpgcheck=1
enable=1
gpgkey=http://mirrors.sohu.com/centos/RPM-GPG-KEY-CentOS-6
[epel]
name=fedora-epel
baseurl=http://mirrors.sohu.com/fedora-epel/$releasever/$basearch/
enable=1
gpgcheck=0　　提示：如果你的系统是centos，系统默认的yum源是可以用的
　　2、拓扑图

　　3、IP地址规划
　　Client: 172.16.254.28
　　Keepalived+Nginx1: 172.16.3.3 Vip: 172.16.3.100
　　Keepalived+Nginx2: 172.16.3.4 Vip: 172.16.3.200
　　

　　二、安装
　　1、安装keepalived
[root@node1 ~]# yum install keepalived　　2、编译安装Nginx
　　
[root@node1 ~]#useradd -r nginx
[root@node1 ~]#yum -y groupinstall "Development tools" "Server  Platform Development"
[root@node1 ~]#yum -y install pcre-devel
[root@node1 ~]#tar xf nginx-1.4.2.tar.gz
[root@node1 ~]#cd nginx-1.4.2
[root@node1 nginx-1.4.2]# ./configure \
--prefix=/usr\
--sbin-path=/usr/sbin/nginx\
--conf-path=/etc/nginx/nginx.conf \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--pid-path=/var/run/nginx/nginx.pid  \
--lock-path=/var/lock/nginx.lock \
--user=nginx \
--group=nginx \
--with-http_ssl_module \
--with-http_flv_module \
--with-http_stub_status_module \
--with-http_gzip_static_module \
--http-client-body-temp-path=/var/tmp/nginx/client/\
--http-proxy-temp-path=/var/tmp/nginx/proxy/\
--http-fastcgi-temp-path=/var/tmp/nginx/fcgi/\
--http-uwsgi-temp-path=/var/tmp/nginx/uwsgi\
--http-scgi-temp-path=/var/tmp/nginx/scgi\
--with-pcre　　提示：在两台服务器上都要安装Nginx
　　3、提供Nginx服务启动脚本
　　
　　
　　
[root@node1 nginx-1.4.2]# vim /etc/rc.d/init.d/nginx
#!/bin/sh
#
# nginx - this script starts and stops the nginx daemon
#
# chkconfig:   - 85 15
# description:  Nginx is an HTTP(S) server, HTTP(S) reverse \
#               proxy and IMAP/POP3 proxy server
# processname: nginx
# config:      /etc/nginx/nginx.conf
# config:      /etc/sysconfig/nginx
# pidfile:     /var/run/nginx.pid
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "$NETWORKING"= "no"] && exit0
nginx="/usr/sbin/nginx"
prog=$(basename$nginx)
NGINX_CONF_FILE="/etc/nginx/nginx.conf"
[ -f /etc/sysconfig/nginx] && . /etc/sysconfig/nginx
lockfile=/var/lock/subsys/nginx
make_dirs() {
# make required directories
user=`nginx -V 2>&1 | grep"configure arguments:"| sed's/[^*]*--user=\([^ ]*\).*/\1/g'-`
options=`$nginx -V 2>&1 | grep'configure arguments:'`
foropt in$options; do
if[ `echo$opt | grep'.*-temp-path'` ]; then
value=`echo$opt | cut-d "="-f 2`
if[ ! -d "$value"]; then
# echo "creating" $value
mkdir-p $value && chown-R $user $value
fi
fi
done
}
start() {
[ -x $nginx ] || exit5
[ -f $NGINX_CONF_FILE ] || exit6
make_dirs
echo-n $"Starting $prog: "
daemon $nginx -c $NGINX_CONF_FILE
retval=$?
echo
[ $retval -eq0 ] && touch$lockfile
return$retval
}
stop() {
echo-n $"Stopping $prog: "
killproc $prog -QUIT
retval=$?
echo
[ $retval -eq0 ] && rm-f $lockfile
return$retval
}
restart() {
configtest || return$?
stop
sleep1
start
}
reload() {
configtest || return$?
echo-n $"Reloading $prog: "
killproc $nginx -HUP
RETVAL=$?
echo
}
force_reload() {
restart
}
configtest() {
$nginx -t -c $NGINX_CONF_FILE
}
rh_status() {
status $prog
}
rh_status_q() {
rh_status >/dev/null2>&1
}
case"$1"in
start)
rh_status_q && exit0
$1
;;
stop)
rh_status_q || exit0
$1
;;
restart|configtest)
$1
;;
reload)
rh_status_q || exit7
$1
;;
force-reload)
force_reload
;;
status)
rh_status
;;
condrestart|try-restart)
rh_status_q || exit0
;;
*)
echo$"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}"
exit2
esac
[root@node1 nginx-1.4.2]# chmod +x /etc/rc.d/init.d/nginx
[root@node1 nginx-1.4.2]# chkconfig --add nginx
[root@node1 nginx-1.4.2]# chkconfig nginx on
[root@node1 nginx-1.4.2]# service nginx start
Starting nginx:                                            [  OK  ]　　提示：服务脚本在两台Nginx服务器上都要提供
　　三、实现高可用

　　1、修改配置keepalived
　　MASTER:172.16.3.3
[root@node1 ~]# cd /etc/keepalived
[root@node1 keepalived]# mv keepalived.conf keepalived.conf.bak     # 备份keepalived配置文件
[root@node1 keepalived]#vim keepalived.conf                     # 创建配置文件，如下
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from kaadmin@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_script chk_nginx {
script "killall -0 nginx"
interval 1
weight 2
}
vrrp_instance VI_10 {
state MASTER
interface eth0
virtual_router_id 131
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.16.3.100
}
track_script {
chk_nginx
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}　　BACKUP:172.16.3.4

　　
[root@node2 ~]# cd /etc/keeplived
[root@node2 keepalived]# mv keepalived.conf keepalived.conf.bak
[root@node2 keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from kaadmin@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_script chk_nginx {
script "killall -0 nginx"
interval 1
weight 2
}
vrrp_instance VI_10 {
state BACKUP
interface eth0
virtual_router_id 131
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.16.3.100
}
track_script {
chk_nginx
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}　　2、服务监控脚本

　　
　　
　　
[root@node1 ~]# vim /etc/keepalived/notify.sh
#!/bin/bash
#
vip=172.16.3.100
contact='root@localhost'
notify() {
mailsubject="`hostname` to be $1: $vip floating"
mailbody="`date '+%F %H:%M:%S'`: vrrp transition,`hostname` changed to be $1"
echo $mailbody | mail -s "$mailsubject" $contact
}
case "$1" in
master)
notify master
/etc/rc.d/init.d/nginx start
exit 0
;;
backup)
notify backup
/etc/rc.d/init.d/nginx restart
exit 0
;;
fault)
notify fualt
exit 0
;;
*)
echo 'Usage: `basename $0` {master|backup|fault}'
exit 1
;;
esac　　复制脚本到另一节点

[root@node1 ~]# scp -p /etc/keepalived/notify.sh root@172.16.3.4:/etc/keepalived/　　3、验证
　　
[root@node1 ~]# ip addr show
1: lo:  mtu 16436 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2:  eth0:  mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
link/ether 00:0c:29:65:a0:2f brd ff:ff:ff:ff:ff:ff
inet 172.16.3.3/16 brd 172.16.255.255 scope global eth0
inet 172.16.3.100/32 scope global eth0          # Vip此时在Keepalived+Nginx1上
inet6 fe80::20c:29ff:fe65:a02f/64 scope link
valid_lft forever preferred_lft forever　　然后关闭Keepalived+Nginx1上的keepalived服务Vip会跳转到Keepalived上
　　四、实现负载均衡
　　1、修改Keepalived配置文件
　　Keepalived+Nginx1：
[root@node1 ~]# cd /etc/keepalived
[root@node1 keepalived]# mv keepalived.conf keepalived.conf.bak     # 备份keepalived配置文件
[root@node1 keepalived]#vim keepalived.conf                         # 创建配置文件，如下
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from kaadmin@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_script chk_nginx {
script "killall -0 nginx"
interval 1
weight 2
}
vrrp_instance VI_10 {
state MASTER
interface eth0
virtual_router_id 131
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.16.3.100
}
track_script {
chk_nginx
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
vrrp_instance VI_20 {
state BACKUP
interface eth0
virtual_router_id 132
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.16.3.200
}
track_script {
chk_nginx
}
notify_master "/etc/keepalived/notify1.sh master"
notify_backup "/etc/keepalived/notify1.sh backup"
notify_fault "/etc/keepalived/notify1.sh fault"
auth_pass 123456
}　　Keepalived+Nginx2
[root@node2 ~]# cd /etc/keeplived
[root@node2 keepalived]# mv keepalived.conf keepalived.conf.bak
[root@node2 keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from kaadmin@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_script chk_nginx {
script "killall -0 nginx"
interval 1
weight 2
}
vrrp_instance VI_10 {
state BACKUP
interface eth0
virtual_router_id 131
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.16.3.100
}
track_script {
chk_nginx
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
vrrp_instance VI_20 {
state MASTER
interface eth0
virtual_router_id 132
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.16.3.200
}
track_script {
chk_nginx
}
notify_master "/etc/keepalived/notify1.sh master"
notify_backup "/etc/keepalived/notify1.sh backup"
notify_fault "/etc/keepalived/notify1.sh fault"
}　　提示：notify1.sh脚本与notify.sh只修改了vip
　　2、重启Keepalived服务
[root@node1 ~]# service keepalived restart
[root@node2 ~]# service keepalived restart　　3、验证
　　
　　
[root@node2 ~]# ip addr show
1: lo:  mtu 16436 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0:  mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
link/ether 00:0c:29:52:ee:44 brd ff:ff:ff:ff:ff:ff
inet 172.16.3.4/16 brd 172.16.255.255 scope global eth0
inet 172.16.3.200/32 scope global eth0    #此时vip2在Keepalived+Nginx2上
inet6 fe80::20c:29ff:fe52:ee44/64 scope link
valid_lft forever preferred_lft forever　　4、关闭Keepalived+Nginx2上的Keepalived服务模拟故障，vip2会转移到Keepalived+Nginx1上，反之依然
　　
[root@node1 ~]# ip addr show
1: lo:  mtu 16436 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0:  mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
link/ether 00:0c:29:65:a0:2f brd ff:ff:ff:ff:ff:ff
inet 172.16.3.3/16 brd 172.16.255.255 scope global eth0
inet 172.16.3.100/32 scope global eth0
inet 172.16.3.200/32 scope global eth0
inet6 fe80::20c:29ff:fe65:a02f/64 scope link
valid_lft forever preferred_lft forever　　至此，此博客结束，欢迎各位博友多提宝贵意见！