生产环境下Tengine keepalived tomcat 安装和配置

超酷小

一，tengine 安装和配置
1.基础库安装：pcre，openssl，geoip，luajit
#mkdir -p /var/log/nginx
#mkdir -p /var/nginx_temp/{nginx_client,nginx_proxy,nginx_fastcgi,nginx_temp}
#cd /data/soft
#wget
#tar zxvf pcre-8.36.tar.gz
#cd pcre-8.36
#./configure --prefix=/usr/local
#make && make install

#cd /data/soft
#wget  
#cd openssl-1.0.1m
./config shared --prefix=/usr/local
#make && make install
#cd /data/soft
#wget
#tar zxvf GeoIP-1.6.5.tar.gz
#cd GeoIP-1.6.5
./configure --prefix=/usr/local
#make && make install
#cd /data/soft
#wget
#tar zxvf LuaJIT-2.0.4.tar.gz
#cd LuaJIT-2.0.4
#make PREFIX=/usr/local
make install  
#cd /data/soft
#tar xvzf tengine-1.5.2.tar.gz
#cd tengine-1.5.2
#./configure --prefix=/opt/nginx \
--lock-path=/var/lock/nginx.lock \
--pid-path=/var/run/nginx.pid \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--user=nobody \
--group=nobody \
--with-pcre=../pcre-8.36 \
--with-pcre-opt=-fPIC \
--with-openssl=../openssl-1.0.1m \
--with-openssl-opt=-fPIC \
--with-backtrace_module \
--with-http_stub_status_module \
--with-http_gzip_static_module \
--with-http_realip_module \
--with-http_concat_module=shared \
--with-http_sysguard_module=shared \
--with-http_limit_conn_module=shared \
--with-http_limit_req_module=shared \
--with-http_split_clients_module=shared \
--with-http_footer_filter_module=shared \
--with-http_geoip_module=shared \
--with-http_sub_module=shared \
--with-http_access_module=shared \
--with-http_upstream_ip_hash_module=shared \
--with-http_upstream_least_conn_module=shared \
--with-http_referer_module=shared \
--with-http_rewrite_module=shared \
--with-http_memcached_module=shared \
--with-http_upstream_session_sticky_module=shared \
--with-http_addition_module=shared \
--with-http_xslt_module=shared \
--with-http_image_filter_module=shared \
--with-http_user_agent_module=shared \
--with-http_empty_gif_module=shared \
--with-http_browser_module=shared \
--with-http_map_module=shared \
--with-http_userid_filter_module=shared \
--with-http_charset_filter_module=shared \
--with-http_trim_filter_module=shared \
--with-http_lua_module=shared \
--without-http_fastcgi_module \
--without-http_uwsgi_module \
--without-http_scgi_module \
--without-select_module \
--without-poll_module \
-with-http_fastcgi_module=shared \
--with-http_uwsgi_module=shared  \
--http-client-body-temp-path=/var/nginx_temp/nginx_client \
--http-proxy-temp-path=/var/nginx_temp/nginx_proxy \
--http-fastcgi-temp-path=/var/nginx_temp/nginx_fastcgi \
--with-cc-opt='-g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security' --with-ld-opt='-Wl,-Bsymbolic-functions -Wl,-z,relro’
# make && make install
#mkdir -p /opt/nginx/conf/{upstream,vhost.d}
#cd /opt
#mv nginx tengine-1.5.2 && ln -s tengine-1.5.2 nginx
#rm -f /opt/nginx/conf/*.default


2.tengine 主配置
#vi /opt/nginx/conf/nginx.conf
user nobody nobody;
worker_processes auto;
worker_cpu_affinity auto;
error_log /var/log/nginx/error.log crit;
pid /var/run/nginx.pid;
worker_rlimit_nofile 65535;
dso {
    load ngx_http_rewrite_module.so;
    load ngx_http_access_module.so;
    load ngx_http_limit_conn_module.so;
    load ngx_http_limit_req_module.so;
    load ngx_http_sysguard_module.so;
    load ngx_http_lua_module.so;
}
events {
    use epoll;
    worker_connections 10240;
}
http {
    server_tokens off;
    server_tag off;
    autoindex off;
    access_log off;
    include mime.types;
    default_type application/octet-stream;
    server_names_hash_bucket_size 128;
    client_header_buffer_size 32k;
    large_client_header_buffers 4 32k;
    client_max_body_size 30m;
    client_body_buffer_size 256k;
    sendfile on;
    tcp_nopush on;
    keepalive_timeout 60;
    tcp_nodelay on;
    gzip on;
    gzip_min_length 1k;
    gzip_buffers 4 16k;
    gzip_http_version 1.1;
    gzip_comp_level 6;
    gzip_types text/plain application/x-javascript text/css application/xml application/javascript text/javascript image/jpeg image/gif image/png;
    gzip_vary on;
    proxy_connect_timeout 120;
    proxy_read_timeout 120;
    proxy_send_timeout 120;
    proxy_buffer_size 128k;
    proxy_buffers 4 128k;
    proxy_busy_buffers_size 256k;
    proxy_temp_file_write_size 256k;
    proxy_headers_hash_max_size 1024;
    proxy_headers_hash_bucket_size 128;
    proxy_redirect off;
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_temp_path /var/nginx_temp/nginx_temp;
  
  
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for" '
                    '$request_time';
log_format ha '$remote_addr\t-\t$remote_user\t[$time_local]\t"$request"\t'
                  '$status $body_bytes_sent\t"$http_referer"\t'
                  '"$http_user_agent"\t$request_time\t'
                  '$cookie_user_id__baidu_com\t$cookie_place_id__baidu_com\t'
                  '$cookie_elatitude\t$cookie_elongitude\t'
                  '$cookie_fixed_cid\t$cookie_temp_cid\t$http_x_requested_with\t$cookie_track_id\t$sent_http_content_type\t'
                  '$http_x_baidu_requestid\t$host';
   
    server {
        sysguard on;
        sysguard_load load=50 action=/loadlimit;
        sysguard_mem swapratio=24% action=/swaplimit;
        sysguard_mem free=100M action=/freelimit;
        location /loadlimit {
            return 503;
        }
        location /swaplimit {
            return 503;
        }
        location /freelimit {
            return 503;
        }
    }
    include upstream/*.conf;
    include vhost.d/*.conf;
}

3. 负载均衡池配置
#vim /opt/nginx/conf/upstream/baidi.com
upstream b_baidi_SERVER {
    consistent_hash $request_uri;
    server 100.10.111.210:8080 id=101 weight=8;
    server 100.10.111.180:8080 id=102 weight=10;
    server 100.10.210.139:8080 id=103 weight=10;
    server 100.10.112.163:8080 id=104 weight=5;
    check interval=3000 rise=2 fall=3 timeout=1000 type=http port=8080;
    check_http_send "OPTIONS http://baidi.com/api/check;
    check_http_expect_alive http_2xx http_3xx;
}
4vhost 设置
#vi /opt/nginx/conf/vhost.d/baidi.com.conf
server {
listen 80;
listen 443 ssl;
server_name baidi.com;
ssl_certificate /opt/nginx/conf/ssl/baidi.crt;
ssl_certificate_key /opt/nginx/conf/ssl/baidi.key;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
set $rewrite_status 0;
if ($https_status = off) {
set $rewrite_status "${rewrite_status}1";
}
if ($scheme = http) {
set $rewrite_status "${rewrite_status}2";
}
if ($https = on) {
set $https_status $https;
}
if ($rewrite_status = 012) {
rewrite / https://$http_host$request_uri permanent;
break;
}
access_log /var/log/nginx/baidi.com.access.log ha;
error_log  /var/log/nginx/baidi.com.error.log;
add_header Access-Control-Allow-Origin *;
add_header Access-Control-Allow-Credentials true;
add_header Access-Control-Allow-Headers Content-Type;
add_header Access-Control-Allow-Methods POST;
location / {
        proxy_next_upstream http_502 http_504 error timeout invalid_header;
        proxy_redirect off;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header REMOTE-HOST $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-For $remote_addr;
proxy_pass http://b_baidi_SERVER;
proxy_set_header Host $host;
}
location ~ health_status {
        check_status;
        allow 127.0.0.1;
        allow 10.0.0.0/8;
        allow 172.0.0.0/8;
        allow 192.168.0.0/16;
        deny all;
    }
}


4 启动脚本
#vim /etc/init.d/nginx
#!/bin/sh
#
# nginx - this script start and stop the nginx daemon
#
# chkconfig: 2345
# description: Startup script for nginx
# processname: nginx
# config: /opt/nginx/conf/nginx.conf
# pidfile: /var/run/nginx.pid
#
# code by rocketzhang
#
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
DAEMON="/opt/nginx/sbin/nginx"
CNFFILE="/opt/nginx/conf/nginx.conf"
PIDFILE="/var/run/nginx.pid"
LOCKFILE="/var/lock/nginx.lock"
RETVAL=0
set -e
[[ -x "$DAEMON" ]] || exit 0
start() {
    [[ -x $DAEMON ]] || exit 5
    [[ -f $CNFFILE ]] || exit 6
    echo -n "Startting Nginx......"
    $DAEMON -c $CNFFILE || echo -n "Nginx already running!"
    RETVAL=$?
    echo
    [[ $RETVAL -eq 0 ]] && touch $LOCKFILE
}
stop() {
    echo -n "Stopping Nginx......"
    if [[ -n `ps aux | grep nginx | awk '/master/{print $2}'` ]]; then
        kill -QUIT `ps aux | grep nginx | awk '/master/{print $2}'`
        RETVAL=$?
        echo
        [[ $RETVAL -eq 0 ]] && rm -f $LOCKFILE $PIDFILE
    fi
}
reload() {
    echo -n "Reloading Nginx......"
    if [[ -n `ps aux | grep nginx | awk '/master/{print $2}'` ]]; then
        kill -HUP `ps aux | grep nginx | awk '/master/{print $2}'`
        RETVAL=$?
        echo
    fi
}
case "$1" in
start)
    start
    ;;
stop)
    stop
    ;;
reload)
    reload
    ;;
restart)
    stop
    sleep 1
    start
    ;;
*)
    echo "Usage: service nginx {start|stop|reload|restart}"
    RETVAL=1
    ;;
esac
exit $RETVAL

5.# 服务启动
#chmod +x /etc/init.d/nginx
#chkconfig nginx on
#service nginx start


主节点配置
二，keepaived 安装和配置
1.基础安装
#yum -y install keepalived

#vim /etc/keepalived/keepalived.conf
global_defs {
    notification_email {
        rickyhui@yahoo.com
    }
    notification_email_from yanming.zhang@ele.me
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id NGX_DEVEL1
}
vrrp_script check_nginx_health {
    script "/opt/scripts/check_nginx_for_keepalived.sh"
    interval 1
    weight -2
}
vrrp_instance VI_1 {
    state MASTER
    interface bond0
    virtual_router_id 51
    priority 100
    advert_int 1
    smtp_alert
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        10.0.20.200
    }
    track_script {
        check_nginx_health
    }
    notify_master "/opt/scripts/notify_for_keepalived.sh master"
    notify_bakcup "/opt/scripts/notify_for_keepalived.sh backup"
    notify_fault "/opt/scripts/notify_for_keepalived.sh fault"
}

备节点配置
#vim /etc/keepalived/keepalived.conf
global_defs {
    notification_email {
        rickyhui@yahoo.com
    }
    notification_email_from rickyhui@yahoo.com
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id NGX_DEVEL2
}
vrrp_script check_nginx_health {
    script "/opt/scripts/check_nginx_for_keepalived.sh"
    interval 1
    weight -2
}
vrrp_instance VI_1 {
    state BACKUP
    interface bond0
    virtual_router_id 51
    priority 90
    advert_int 1
    smtp_alert
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        10.0.20.200
    }
    track_script {
        check_nginx_health
    }
    notify_master "/opt/scripts/notify_for_keepalived.sh master"
    notify_bakcup "/opt/scripts/notify_for_keepalived.sh backup"
    notify_fault "/opt/scripts/notify_for_keepalived.sh fault"
}

#脚本
#vim /opt/scripts/check_nginx_for_keepalived.sh
#!/bin/sh
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin
if [[ -z `ps aux | grep 'nginx:' | grep -v grep` ]]; then
    /etc/init.d/nginx start && sleep 5
    if [[ -z `ps aux | grep 'nginx:' | grep -v grep` ]]; then
        /sbin/service keepalived stop
    fi
fi
VIP 漂移短信通知脚本
# vim /opt/scripts/notify_for_keepalived.sh
#!/bin/sh
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin
KEEPALIVE_CONF="/etc/keepalived/keepalived.conf"
VIP=`grep -A 1 virtual_ipaddress ${KEEPALIVE_CONF} | tail -1 | sed 's/\t//g; s/ //g'`
## 短信发送
API_KEY="xxxxxxx"
URL="