nginx+Keepalived双机主备方案实现（经线上测试）

wer21213

                                                                  

• nginx的编译安装
  
  

1）编译工具（基本可以不用）
    yum -y install gcc gcc-c++ autoconf automake zlib zlib-devel openssl openssl-devel pcre-devel gd-*
2）建立nginx 组
    groupadd -r nginx
    useradd -s /sbin/nologin -g nginx -r nginx
    id nginx
3）wget http://nginx.org/download/nginx-1.6.2.tar.gz
    tar xf /usr/local/src/nginx-1.6.2.tar.gz
    cd /usr/local/src/nginx-1.6.2.tar.gz
./configure --prefix=/usr/local/nginx --lock-path=/usr/local/nginx/nginx.lock --user=nginx --group=nginx --with-http_ssl_module --with-http_flv_module --with-http_stub_status_module --with-http_gzip_static_module --http-client-body-temp-path=/usr/local/nginx/client/ --http-proxy-temp-path=/usr/local/nginx/proxy/ --http-fastcgi-temp-path=/usr/local/nginx/fcgi/ --http-uwsgi-temp-path=/usr/local/nginx/uwsgi --http-scgi-temp-
path=/usr/local/nginx/scgi --with-pcre --with-file-aio --with-http_image_filter_module

   make && make install
4)配置nginx启动脚本
   ===============================================
vim /etc/init.d/nginx   chmod +x /etc/init.d/nginx  
#!/bin/bash
#
# Startup script for Nginx - this script starts and stops the nginx daemon
#
# chkconfig:   - 85 15
# description:  Nginx is an HTTP(S) server, HTTP(S) reverse proxy and IMAP/POP3 proxy server
# processname: nginx
# config:      /usr/local/nginx/conf/nginx.conf
# pidfile:     /usr/local/nginx/logs/nginx.pid
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "$NETWORKING" = "no" ] && exit 0
nginx="/usr/local/nginx/sbin/nginx"
prog=$(basename $nginx)
NGINX_CONF_FILE="/usr/local/nginx/conf/nginx.conf"
[ -f /etc/sysconfig/nginx ] && . /etc/sysconfig/nginx
lockfile=/var/lock/subsys/nginx
start() {
    [ -x $nginx ] || exit 5
    [ -f $NGINX_CONF_FILE ] || exit 6
    echo -n $"Starting $prog: "
    daemon $nginx -c $NGINX_CONF_FILE
    retval=$?
    echo
    [ $retval -eq 0 ] && touch $lockfile
    return $retval
}

stop() {
    echo -n $"Stopping $prog: "
    killproc $prog -QUIT
    retval=$?
    echo
    [ $retval -eq 0 ] && rm -f $lockfile
    return $retval
}

restart() {
    configtest || return $?
    stop
    sleep 1
    start
}

reload() {
    configtest || return $?
    echo -n $"Reloading $prog: "
    killproc $nginx -HUP
    RETVAL=$?
    echo
}

force_reload() {
    restart
}

configtest() {
  $nginx -t -c $NGINX_CONF_FILE
}

rh_status() {
    status $prog
}

rh_status_q() {
    rh_status >/dev/null 2>&1
}

case "$1" in
    start)
        rh_status_q && exit 0
        $1
        ;;
    stop)
        rh_status_q || exit 0
        $1
        ;;
    restart|configtest)
        $1
        ;;
    reload)
        rh_status_q || exit 7
        $1
        ;;
    force-reload)
        force_reload
        ;;
    status)
        rh_status
        ;;
    condrestart|try-restart)
        rh_status_q || exit 0
            ;;
    *)
        echo $"Usage: $0 {start|stop|status|restart|condrestart|

try-restart|reload|force-reload|configtest}"
        exit 2
esac

==============================================================
5)nginx配置
vim /usr/local/nginx/conf/nginx.conf
====================================================
upstream MCM {
       server 10.124.156.236; #使用nginx的默认分发模式rr
       server 10.124.156.231; #两个分发服务器ip
     }

server {
        listen       80;
        server_name  localhost;
        location /mcm/ {
limit_req zone=allips burst=10 ;
proxy_pass http://MCM/mcm/;
proxy_redirect  off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For

$proxy_add_x_forwarded_for;
proxy_set_header Cookie $http_cookie;
chunked_transfer_encoding  off;
        }
         error_page   500 502 503 504  /50x.html;
         location = /50x.html {
             root   html;
         }
  =========================================================
2.Keepalived的安装配置
1)Keepalived的编译安装
  cd /usr/local/src/
  wget http://www.keepalived.org/software/keepalived-1.1.20.tar.gz
  tar xf keepalived-1.2.15.tar.gz
  cd keepalived-1.1.20
  ./configure --prefix=/usr/local/keepalived --with-kernel-dir=/usr/src/kernels/2.6.32-279.el6.x86_64/
  make && make install
  将Keepalived做成服务：
   cd /usr/local/keepalived/ && cp etc/rc.d/init.d/keepalived /etc/rc.d/init.d/ && cp etc/sysconfig/keepalived /etc/sysconfig/ && mkdir /etc/keepalived && cp etc/keepalived/keepalived.conf /etc/keepalived/ && cp sbin/keepalived /usr/sbin/

2)配置Keepalived.conf文件
   ====================================================
   TestNginx1：10.124.194.244
   vim /etc/keepalived/keepalived.conf

   ==========================================
! Configuration File for keepalived

global_defs {
   notification_email {
       459202366@139.com  
}
   notification_email_from Keepalived@chtopnet.com
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id Nginx_MB
}

vrrp_script chk_http_port {
    script "/etc/keepalived/chk_nginx.sh"
    interval 2
    weight 2
}

vrrp_instance VI_3 {
    state MASTER
    interface eth0
    virtual_router_id 57      #VI_2 为 58
    priority 200              #备用是150
    mcast_src_ip 10.124.194.244 #本机ip
    advert_int 1

authentication {
        auth_type PASS
        auth_pass yzkj
    }
    track_script {
        chk_http_port
    }
    virtual_ipaddress {
        10.124.194.4         #VIP 1
    }
}
vrrp_instance VI_4 {
    state BACKUP
    interface eth0
    virtual_router_id 58     # VI_1 是57
    priority 150            #主是200
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass yzkj
    }
    track_script {
        chk_http_port
    }
    virtual_ipaddress {
        10.124.194.5       #VIP 2
    }
}
========================================
TestNginx2

! Configuration File for keepalived

global_defs {
   notification_email {
       459202366@139.com  
}
   notification_email_from Keepalived@chtopnet.com
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id Nginx_MB
}

vrrp_script chk_http_port {
    script "/etc/keepalived/chk_nginx.sh"
    interval 2
    weight 2
}

vrrp_instance VI_3 {
    state BACKUP
    interface eth0
    virtual_router_id 57        #VI_4 为 58
    priority  150            #主是200
    advert_int 1

authentication {
        auth_type PASS
        auth_pass yzkj
    }
    track_script {
        chk_http_port
    }
    virtual_ipaddress {
        10.124.194.4         #VIP 1
    }
}
vrrp_instance VI_4 {
    state MASTER
    interface eth0
    virtual_router_id 58     #VI_3 是57
    priority 200             #备用是50
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass yzkj
    }
    track_script {
        chk_http_port
    }
    virtual_ipaddress {
        10.124.194.5       #VIP 2
    }
}
=====================================
3)添加nginx状态自动检测脚本
  vim /etc/keepalived/chk_nginx.sh
=============  
#!/bin/bash
# description:
# 定时查看nginx是否存在，如果不存在则启动nginx
# 如果启动失败，则停止keepalived
status=$(ps -C nginx --no-heading|wc -l)
if [ "${status}" = "0" ]; then
        /usr/local/nginx/sbin/nginx
        status2=$(ps -C nginx --no-heading|wc -l)
        if [ "${status2}" = "0"  ]; then
                /etc/init.d/keepalived stop
        fi
fi
====================
4)keepalived nginx启动
service keepalived start
service nginx start  或者（/usr/local/nginx/sbin/nginx）

5）防火墙设置：
  在Lvs+Keepalived的设置中一般都将iptables关掉，这是一个很危险的操作。在测试nginx+Keepalived的时候发现iptables不关闭的话，会出现VIP同时绑定在主备nginx上。所以经过研究测试，要想开着iptables来使用Keepalived，就应该先明白Keepalived的工作原理。Keepalived工作主
要是通过vrrp协议进行报文互通，于是就必须在iptables里设置
-A INPUT -p vrrp -j ACCEPT #基于DR模式，当用户发出请求后，只有DR响应ARP广播包，允许vrrp虚拟路由器冗余协议。然后重启iptables服务，至此困扰已久的VIP同时绑定在多个服务器上的问题解决。
3.测试：
在测试过程中出现一个有趣的话题：就是当我进行停止nginx运行的时候，发现进程自动会起来，然后我就问、查，总是不以所以然，总以为是kill pkill命令不好使。后来经过查证，竟然是我们写的一个检测脚本在作怪。脚本记录如下：（修改完善版）
  ===================================
#!/bin/bash
# description:
# 定时查看nginx是否存在，如果不存在则启动nginx
# 如果启动失败，则停止keepalived
status=$(ps -C nginx --no-heading|wc -l)
if [ "${status}" = "0" ]; then
        /usr/local/nginx/sbin/nginx
        status2=$(ps -C nginx --no-heading|wc -l)
        status3=$(ps -C keepalived --no-heading|wc -l)
        if [ "${status2}" = "0"  ]; then
                /etc/init.d/keepalived stop
        elif [ "${status3}" = "0" ]; then
                /etc/init.d/keepalived start
             fi
        fi
fi
========================================
它的作用主要就是检测nginx的状态，如果nginx死掉，就尝试重启，如果重启不成功就把Keepalived服务停止，使两个vip同时漂到同一台nginx服务器上面去，如果nginx恢复就应该重启Keepalived服务。
  4.使用rsync使后端几台服务器文件同步
1）服务安装：yum install rsync -y
    服务端：10.124.151.246
    =============================
vim /etc/rsyncd.conf
uid = root
gid = root
use chroot = no
max connections = 5
pid file = /var/run/rsyncd.pid
lock file = /var/run/rsync.lock
log file = /var/log/rsyncd.log
[web01]                     
path=/usr/local/nginx/html/         
comment = update         
ignore errors              
read only = no            
list = no                 
hosts allow = 192.168.122.0/24
auth users = root        
uid = root
gid = root
secrets file = /etc/rsyncd.secrets
    =================================
vim /etc/rsyncd.secrets
root:123456

chmod 0600 /etc/rsyncd.secrets
    =============================
  启动服务：rsync --daemon
            echo "rsync --daemon" >> /etc/rc.local #自启动

  客户端配置（也就是代码推送端）10.124.151.245
    vim /etc/rsyncd.secrets
    123456
    chmod 0600 /etc/rsyncd.secrets
2)使用命令行进行代码推送（根据代码的修改和增删情况进行代码同步操作）
   rsync -vzrtopg --delete --progress --password-file=/etc/rsyncd.secrets  * root@10.124.151.246::web01