Nginx+Keepalived主备模式配置文档-jesse于2011-0920

edcf

　　
环境:
centos5.5x64
nginx-0.8.53.tar.gz
keepalived-1.1.17.tar.gz
pcre-8.11.tar.gz
　　定义如下:
nginx-master:10.0.0.15
nginx-backup:10.0.0.18
vip:10.0.0.200
#######################################
mkdir /tools
cd /tools
wget http://nginx.org/download/nginx-0.8.53.tar.gz
wget http://www.keepalived.org/software/keepalived-1.1.17.tar.gz
wget http://ftp.exim.llorien.org/pcre/pcre-8.11.tar.gz
　　一:分别安装Nginx负载均衡器及相关配置脚本
　　添加运行nginx的用户和组www
　　groupadd www
useradd -g www www
　　tar zxvf pcre-8.11.tar.gz
cd pcre-8.11/
./configure
make
make install
cd ..
　　tar zxvf nginx-0.8.53.tar.gz
cd nginx-0.8.53/
./configure --user=www \
--group=www \
--prefix=/usr/local/nginx-0.8.53 \
--with-http_stub_status_module \
--with-http_ssl_module
make
make install
cd ..
　　vi nginx.conf
#-----------------------------------------------------------------------------------
user www www;
worker_processes 8;
pid /usr/local/nginx-0.8.53/logs/nginx.pid;
worker_rlimit_nofile 65535;
　　events
{
use epoll;
worker_connections 65535;
}
http{
include       mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 8m;
user www www;
worker_processes 8;
pid /usr/local/nginx-0.8.53/logs/nginx.pid;
worker_rlimit_nofile 65535;
　　events
{
use epoll;
worker_connections 65535;
}
　　http{
include                       mime.types;
default_type                  application/octet-stream;
server_names_hash_bucket_size 128;
client_header_buffer_size     32k;
large_client_header_buffers   4 32k;
client_max_body_size          8m;
sendfile                      on;
tcp_nopush                    on;
keepalive_timeout             60;
tcp_nodelay                   on;
fastcgi_connect_timeout       300;
fastcgi_send_timeout          300;
fastcgi_read_timeout          300;
fastcgi_buffer_size           64k;
fastcgi_buffers               4 64k;
fastcgi_busy_buffers_size     128k;
fastcgi_temp_file_write_size  128k;
gzip                          on;
gzip_min_length               1k;
gzip_buffers                  4 16k;
gzip_http_version             1.0;
gzip_comp_level               2;
gzip_types                    text/plain application/x-javascript text/css application/xml;
gzip_vary                     on;
　　upstream jesse
{
server 10.0.0.15:700;
server 10.0.0.15:800;
server 10.0.0.15:900;
}
　　server
    {
     listen  80;
     server_name  www.jesse.com;
     location / {
                proxy_pass         http://jesse;
                proxy_set_header   Host             $host;
                proxy_set_header   X-Real-IP        $remote_addr;
                proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
               }
               log_format access  '$remote_addr - $remote_user [$time_local] $request'
               '"$status" $body_bytes_sent "$http_referer" '
               '"$http_user_agent" "$http_x_forwarded_for"';
               access_log  /data/logs/jesse.log  access;
      }
}
#-----------------------------------------------------------------------------------
　　开启转发功能
sed -i 's#net.ipv4.ip_forward = 0#net.ipv4.ip_forward = 1#' /etc/sysctl.conf
sysctl -p
　　二:安装keepalived
tar zxvf keepalived-1.1.17.tar.gz
cd keepalived-1.1.17
./configure --prefix=/usr/local/keepalived-1.1.17
make
make install
cd ..
cp /usr/local/keepalived-1.1.17/sbin/keepalived /usr/sbin/
cp /usr/local/keepalived-1.1.17/etc/sysconfig/keepalived /etc/sysconfig/
cp /usr/local/keepalived-1.1.17/etc/rc.d/init.d/keepalived /etc/init.d/
mkdir /etc/keepalived
cd /etc/keepalived/
　　vi keepalived.conf
#################################[MASTER]
! Configuration File for keepalived
global_defs {
    notification_email {
    64125568@qq.com
         }
    notification_email_from keepalived@chtopnet.com
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id LVS_DEVEL
}
　　vrrp_instance VI_1 {
     state MASTER
     interface eth0
     virtual_router_id 51
     #define nginx-master-IP
     mcast_src_ip 10.0.0.15 #这里是本机nginx-master-IP
     priority 100
     advert_int 1
     authentication {
     auth_type PASS
     auth_pass jesse
     }
     
     #define vip
     virtual_ipaddress {
             10.0.0.200   #这个是VIP
                       }
               }
####################################
/etc/init.d/keepalived start
echo "/etc/init.d/keepalived start" >>/etc/rc.local
　　我们来看一下日志:
[iyunv@MASTER keepalived]# tail /var/log/messages
Sep 20 23:12:40 MASTER Keepalived_vrrp: Registering Kernel netlink reflector
Sep 20 23:12:40 MASTER Keepalived_vrrp: Registering Kernel netlink command channel
Sep 20 23:12:40 MASTER Keepalived_vrrp: Registering gratutious ARP shared channel
Sep 20 23:12:42 MASTER Keepalived_vrrp: Opening file '/etc/keepalived/keepalived.conf'.
Sep 20 23:12:42 MASTER Keepalived_vrrp: Configuration is using : 62588 Bytes
Sep 20 23:12:42 MASTER Keepalived_vrrp: VRRP sockpool: [ifindex(2), proto(112), fd(9,10)]
Sep 20 23:12:43 MASTER Keepalived_vrrp: VRRP_Instance(VI_1) Transition to MASTER STATE
Sep 20 23:12:44 MASTER Keepalived_vrrp: VRRP_Instance(VI_1) Entering MASTER STATE
Sep 20 23:12:44 MASTER Keepalived_vrrp: VRRP_Instance(VI_1) setting protocol VIPs.
Sep 20 23:12:44 MASTER Keepalived_vrrp: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth0 for 10.0.0.200
　　命令检查vrrp是否启动
[iyunv@MASTER keepalived]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 00:0c:29:16:9f:16 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.15/24 brd 10.0.0.255 scope global eth0
    inet 10.0.0.200/32 scope global eth0
    inet6 fe80::20c:29ff:fe16:9f16/64 scope link
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 00:0c:29:16:9f:20 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.26/24 brd 10.0.0.255 scope global eth1
    inet6 fe80::20c:29ff:fe16:9f20/64 scope link
       valid_lft forever preferred_lft forever
4: sit0: <NOARP> mtu 1480 qdisc noop
    link/sit 0.0.0.0 brd 0.0.0.0
　　说明vip已经启动
　　
BACKUP机配置如下:
　　vi keepalived.conf
###################################[BACKUP]
! Configuration File for keepalived
global_defs {
    notification_email {
    64125568@qq.com
       }
       notification_email_from keepalived@chtopnet.com
       smtp_server 127.0.0.1
       smtp_connect_timeout 30
       router_id LVS_DEVEL
      }
　　vrrp_instance VI_1 {
     state BACKUP
     interface eth0
     virtual_router_id 51
     #define nginx-backup-ip
     mcast_src_ip 10.0.0.18
     priority 99
     advert_int 1
     authentication {
     auth_type PASS
     auth_pass jesse
    }
virtual_ipaddress {
         10.0.0.200
     }
}
###################################[BACKUP]###########################################
/etc/init.d/keepalived start
echo "/etc/init.d/keepalived start" >>/etc/rc.local
　　三:针对Keepalived的不足,用check_nginx.sh来监控nginx进程，实现真正意义上的负载均衡高可用.
此脚本思路其实也很简单，即放置在后台一直监控nginx进程；如进程消失，尝试重启nginx，
如是失败则立即停掉本机的keepalived服务，让另一台负载均衡器接手.
　　
vi /shell/nginx_pid.sh
###################################################
#!/bin/bash
while :
    do
    nginxpid=`ps -C nginx --no-header | wc -l`
　　if [ $nginxpid -eq 0 ]
   then
   /usr/local/nginx-0.8.53/sbin/nginx
   sleep 5
if [ $nginxpid -eq 0 ]
    then
    /etc/init.d/keepalived stop
    fi
    fi
    sleep 5
done
###################################################
nohup /bin/bash /shell/nginx_pid.sh &
　　如果你正在运行一个进程，而且你觉得在退出帐户时该进程还不会结束，
那么可以使用nohup命令。该命令可以在你退出root帐户之后继续运行相应的进程.
　　
　　###################################
WEB服务器部署:可以配置多端口的apache服务来测试,以为是测试，这里我就只安装apache了
10.0.0.15:700 显示内容:700
10.0.0.15:800 显示内容:800
10.0.0.15:900 显示内容:900
　　wget http://mirror.bjtu.edu.cn/apache//httpd/httpd-2.2.21.tar.gz
　　tar zvfx httpd-2.2.21.tar.gz
cd httpd-2.2.21
./configure --prefix=/usr/local/apache2.2.21 \
--enable-deflate \
--enable-headers \
--enable-modules=so \
--enable-so \
--enable-rewrite \
--enable-cgi \
--enable-file-cache \
--enable-cache \
--enable-disk-cache \
--enable-mem-cache
make
make install
cd ..
echo "/usr/local/apache2.2.21/bin/apachectl start" >> /etc/rc.local
注意，此处我采用的是perfork模式
　　useradd apache -M -s /sbin/nologin
　　vi /usr/local/apache2.2.21/conf/httpd.conf(做以下修改)
#-----------------------------------------------------------------------------------
1:默认用户和组daemon修改为apache
　　66 User apache
67 Group apache
　　2:注释掉80端口，因为nginx在使用这个
40 #Listen 80
　　3:将如下
105 DocumentRoot "/usr/local/apache2.2.21/htdocs"
132 <Directory "/usr/local/apache2.2.21/htdocs">
　　中的/usr/local/apache2.2.20/htdocs替换成/data
　　4:将371行打开并调优perfork模式
371 #Include conf/extra/httpd-mpm.conf
　　<IfModule mpm_prefork_module>
    StartServers          10
    MinSpareServers       10
    MaxSpareServers       15
    ServerLimit           2000
    MaxClients            1000
    MaxRequestsPerChild   5000
</IfModule>
　　五:打开虚拟主机
394 Include conf/extra/httpd-vhosts.conf
　　6:将次做如下修改:目的是禁止显示站点目录功
<Directory "/data">
Options -Indexes FollowSymLinks
AllowOverride None
Order allow,deny
Allow from all
</Directory>
　　
/usr/local/apache2.2.21/bin/apachectl -t
/usr/local/apache2.2.21/bin/apachectl stop
/usr/local/apache2.2.21/bin/apachectl start
#-----------------------------------------------------------------------------------
　　vi httpd-vhosts.conf
###################################
NameVirtualHost *:700
Listen 700
Listen 800
Listen 900
<VirtualHost *:700>
    ServerAdmin 64125568@qq.com
    DocumentRoot "/data/700"
    ServerName 700.com
    ServerAlias www.700.com
    ErrorLog "logs/700.com-error_log"
    CustomLog "logs/700.com-access_log" common
</VirtualHost>
　　NameVirtualHost *:800
<VirtualHost *:800>
    ServerAdmin 64125568@qq.com
    DocumentRoot "/data/800"
    ServerName 800.com
    ServerAlias www.800.com
    ErrorLog "logs/800.com-error_log"
    CustomLog "logs/800.com-access_log" common
</VirtualHost>
　　NameVirtualHost *:900
<VirtualHost *:900>
    ServerAdmin 64125568@qq.com
    DocumentRoot "/data/900"
    ServerName 900.com
    ServerAlias www.900.com
    ErrorLog "logs/900.com-error_log"
    CustomLog "logs/900.com-access_log" common
</VirtualHost>
#######################################
　　[iyunv@MASTER extra]# mkdir /data/{700,800,900}
[iyunv@MASTER extra]# /usr/local/apache2.2.21/bin/apachectl -t
Syntax OK
　　echo "700" >/data/700/index.html
echo "800" >/data/800/index.html
echo "900" >/data/900/inhtmldex.
　　/usr/local/apache2.2.21/bin/apachectl start
　　访问测试：
down 掉nginx-master
ip add 发现vip 已经不存在了
　　备机执行: ip add
发现 vip 已经飘过来了
　　http://10.0.0.200 ctrl+f5强制刷新,发现每刷一次，出现的内容各不一样。
　　至此，nginx+keepalived的负载均衡主备模式配置成功！