fastdfs负载均衡
环境:centos6.5tracker_server_1 ip:192.168.2.190tracker_server_2 ip:192.168.2.191
storage_server_group1_1 ip: 192.168.2.192storage_server_group1_2 ip:192.168.2.193
storage_server_group2_1 ip: 192.168.2.194storage_server_group2_2 ip:192.168.2.195
第一部分六节点做时间同步
#service iptables stop
#yum install -y ntpdate
#cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
#ntpdate us.pool.ntp.org
时间定时同步(6节点安装)
#crontab -e
输入以下代码,每10分钟同步一次
*/10 * * * * /usr/sbin/ntpdate us.pool.ntp.org | logger -t NTP
第二部分六节点安装FastDFS_v5.05
1、安装相关倚赖包并建立相关文件夹及权限设定
#mkdir /main/soft
所有相关软件上传至/main/soft/
#yum -y install gcc gcc-c++ openssl openssl-devel pcre-devel perl unzip
#/usr/sbin/groupadd www
#/usr/sbin/useradd -s /sbin/nologin -M -g www www
#mkdir -p /main/web/fdfs
#chmod 777 /main/web/fdfs
#mkdir /main/web/fdfs/storage
#chown -R www:www /main/web/fdfs
#chmod g+s /main/web/fdfs
#mkdir -p /main/web/logs
#chmod +w /main/web/logs
#chown -R www:www /main/web/logs
2、安装libfastcommon,FastDFS_v5.05依赖libfastcommon,不再依赖libevent
#cd /main/soft/
#unzip libfastcommon-master.zip
#cd libfastcommon-master
#./make.sh
#./make.sh install
libfastcommon.so默认安装到了/usr/lib64/libfastcommon.so,而FastDFS主程序设置的lib目录是/usr/local/lib,所以设置软连接
#ln -s /usr/lib64/libfastcommon.so /usr/local/lib/libfastcommon.so
#ln -s /usr/lib64/libfastcommon.so /usr/lib/libfastcommon.so
#ln -s /usr/lib64/libfdfsclient.so /usr/local/lib/libfdfsclient.so
#ln -s /usr/lib64/libfdfsclient.so /usr/lib/libfdfsclient.so
3、安装fastdfs
#cd /main/soft
#tar -zxvf fastdfs-5.05.tar.gz
#cd fastdfs-5.05/
#./make.sh
#./make.sh install
第三部分二个tracker节点配置
1、建立数据及日志目录
#mkdir /main/web/fdfs/tracker
2、拷贝及修改tracker配置文件
#cp /etc/fdfs/tracker.conf.sample /etc/fdfs/tracker.conf
#vi /etc/fdfs/tracker.conf
一般只需改动以下几个参数即可:
disabled=false #启用配置文件
port=22122 #设置tracker的端口号
base_path=/main/web/fdfs/tracker #设置tracker的数据文件和日志目录(需预先创建)
http.server_port=8080 #设置http端口号
3、启动并验证tracker节点
#/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf
#echo '/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf' >> /etc/rc.local
启动tracker,确认启动是否成功。(查看是否对应端口22122是否开始监听)
#netstat -unltp|grep fdfs
也可查看tracker的日志是否启动成功或是否有错误。
#cat /main/web/fdfs/tracker/logs/trackerd.log
第四部分storage四节点配置
1、防火墙设置
端口开放
#iptables -I INPUT -p tcp --dport 8080 -j ACCEPT
#iptables -I INPUT -p tcp --dport 22122 -j ACCEPT
#/etc/rc.d/init.d/iptables save
2、拷贝及修改storage配置文件
#cp /etc/fdfs/storage.conf.sample /etc/fdfs/storage.conf
#vi /etc/fdfs/storage.conf
一般只需改动以下几个参数即可:
disabled=false #启用配置文件
group_name=group1 #组名,根据实际情况修改
port=23000 #设置storage的端口号
base_path=/main/web/fdfs/storage #设置storage的日志目录(需预先创建)
store_path_count=1 #存储路径个数,需要和store_path个数匹配
store_path0=/main/web/fdfs/storage #存储路径
tracker_server=192.168.2.190:22122#tracker服务器的IP地址和端口号
tracker_server=192.168.2.191:22122#tracker服务器的IP地址和端口号
http.server_port=8080 #设置http端口号
3、启动storage服务
#/usr/bin/fdfs_storaged /etc/fdfs/storage.conf
重启storage服务
#/usr/bin/restart.sh/usr/bin/fdfs_storaged /etc/fdfs/storage.conf
#cat /main/web/fdfs/storage/logs/storaged.log
确认启动成功后,可以运行fdfs_monitor查看storage服务器是否已经登记到tracker服务器,特别注意每个group中所有storage的端口号必须一致
#/usr/bin/fdfs_monitor /etc/fdfs/storage.conf
第五部分storage四节点上安装NGINX,在storage上安装的nginx主要为了提供http的访问服务,同时解决group中storage服务器的同步延迟问题。(需要安装ngx_cache_purge-2.3.tar.gz)
1、在storage server安装nginx模块
#cd /main/soft/
#tar -zxvf fastdfs-nginx-module_v1.16.tar.gz
#vi /main/soft/fastdfs-nginx-module/src/config
修改如下配置,我这里原来是
CORE_INCS="$CORE_INCS /usr/local/include/fastdfs /usr/local/include/fastcommon/"
改成
CORE_INCS="$CORE_INCS /usr/include/fastdfs /usr/include/fastcommon/"
这个是很重要的,不然在nginx编译的时候会报错的,我看网上很多在安装nginx的fastdfs的插件报错,都是这个原因,而不是版本不匹配。
2、安装Nginx所需的pcre库
#cd /main/soft
#tar zxvf pcre-8.36.tar.gz
#cd pcre-8.36/
#./configure
#make && make install
3、安装nginx及修改配置
#cd /main/soft/
#tar -zxvf nginx-1.6.3.tar.gz
#cd nginx-1.6.3
#./configure --user=www --group=www --prefix=/main/server/nginx --with-http_stub_status_module --with-http_ssl_module --with-http_sub_module --with-pcre=../pcre-8.36 --with-pcre-jit --add-module=../fastdfs-nginx-module/src
#make && make install
4、修改nginx配置
#rm -rf /main/server/nginx/conf/nginx.conf
#vi /main/server/nginx/conf/nginx.conf
复制以下代码
userwww www;
worker_processes4;
error_log/main/web/logs/nginx_error.logcrit;
#error_loglogs/error.log;
#error_loglogs/error.lognotice;
#error_loglogs/error.loginfo;
pid logs/nginx.pid;
events {
use epoll;
worker_connections 65535;
}
http{
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 300m;
include mime.types;
default_typeapplication/octet-stream;
sendfile on;
tcp_nopush on;
proxy_redirect off;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
proxy_buffer_size 16k;
proxy_buffers 4 64k;
proxy_busy_buffers_size 128k;
proxy_temp_file_write_size 128k;
keepalive_timeout65;
server {
listen 8080;
server_namelocalhost;
#charset koi8-r;
#access_loglogs/host.access.logmain;
location~/group/M00{
root /main/web/fdfs/storage/data;
ngx_fastdfs_module;
}
}
log_formataccess'$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $http_x_forwarded_for';
access_log/main/web/logs/access.logaccess;
}
5、复制fastdfs相关文件到配置目录并修改
#cp /main/soft/fastdfs-5.05/conf/http.conf /etc/fdfs/
#cp /main/soft/fastdfs-5.05/conf/mime.types /etc/fdfs/
#cp /main/soft/fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/mod_fastdfs.conf
#vi /etc/fdfs/mod_fastdfs.conf
一般只需改动以下几个参数即可:
base_path=/main/web/fdfs/storage #保存日志目录
tracker_server=192.168.2.190:22122 #tracker服务器的IP地址以及端口号
tracker_server=192.168.2.191:22122 #tracker服务器的IP地址以及端口号
storage_server_port=23000 #storage服务器的端口号
group_name=group1 #当前服务器的group名
url_have_group_name= true #文件url中是否有group名
store_path_count=1 #存储路径个数,需要和store_path个数匹配
store_path0=/main/web/fdfs/storage #存储路径
group_count= 2 #设置组的个数
在末尾增加2个组的具体信息:
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/main/web/fdfs/storage
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/main/web/fdfs/storage
6、给storage的存储目录做一个软连接
# ln -s /main/web/fdfs/storage/data /main/web/fdfs/storage/data/M00
7、编写nginx 启动服务
#vi /etc/init.d/nginx
输入以下代码并保存(4台storage服务器配置)
#!/bin/sh
#
# nginx - this script starts and stops the nginx daemin
#
# chkconfig:- 85 15
# description:Nginx is an HTTP(S) server, HTTP(S) reverse \
# proxy and IMAP/POP3 proxy server
# processname: nginx
# config: /main/server/nginx/conf/nginx.conf
# pidfile: /main/server/nginx/logs/nginx.pid
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "$NETWORKING" = "no" ] && exit 0
nginx="/main/server/nginx/sbin/nginx"
prog=$(basename $nginx)
NGINX_CONF_FILE="/main/server/nginx/conf/nginx.conf"
lockfile=/var/lock/subsys/nginx
start() {
[ -x $nginx ] || exit 5
[ -f $NGINX_CONF_FILE ] || exit 6
echo -n $"Starting $prog: "
daemon $nginx -c $NGINX_CONF_FILE
retval=$?
echo
[ $retval -eq 0 ] && touch $lockfile
return $retval
}
stop() {
echo -n $"Stopping $prog: "
killproc $prog -QUIT
retval=$?
echo
[ $retval -eq 0 ] && rm -f $lockfile
return $retval
}
restart() {
configtest || return $?
stop
start
}
reload() {
configtest || return $?
echo -n $"Reloading $prog: "
killproc $nginx -HUP
RETVAL=$?
echo
}
configtest() {
$nginx -t -c $NGINX_CONF_FILE
}
rh_status() {
status $prog
}
rh_status_q() {
rh_status >/dev/null 2>&1
}
case "$1" in
start)
rh_status_q && exit 0
$1
;;
stop)
rh_status_q || exit 0
$1
;;
restart|configtest)
$1
;;
reload)
rh_status_q || exit 7
$1
;;
status)
rh_status
;;
*)
echo $"Usage: $0 {start|stop|status|restart|reload|configtest}"
exit 2
esac
保存后,设置权限并添加到启动服务列表中
#chmod 755 /etc/init.d/nginx
#chkconfig --add nginx
#chkconfig --level 345 nginx on
#service nginx start
在error.log中没有错误,既启动成功。可以打开浏览器,直接访问http://192.168.2.192:8080,查看是否弹出nginx欢迎页面。
第六部分tracker两节点上安装nginx,在tracker上安装的nginx主要为了提供http访问的反向代理、负载均衡以及缓存服务。
1、建立相关文件夹
#mkdir -p /main/web/cache/nginx/proxy_cache/tmp
#chown -R www:www /main/web/cache
#chmod g+s /main/web/cache
#mkdir /etc/keepalived
2、解压fastdfs-nginx-module模块
#cd /main/soft/
#tar -zxvf fastdfs-nginx-module_v1.16.tar.gz
#vi /main/soft/fastdfs-nginx-module/src/config
修改如下配置,我这里原来是
CORE_INCS="$CORE_INCS /usr/local/include/fastdfs /usr/local/include/fastcommon/"
改成
CORE_INCS="$CORE_INCS /usr/include/fastdfs /usr/include/fastcommon/"
这个是很重要的,不然在nginx编译的时候会报错的,我看网上很多在安装nginx的fastdfs的插件报错,都是这个原因,而不是版本不匹配。
3、解压ngx_cache_purge模块
#tar zxvf ngx_cache_purge-2.3.tar.gz
4、安装Nginx所需的pcre库
#tar zxvf pcre-8.36.tar.gz
#cd pcre-8.36/
#./configure
#make && make install
5、安装nginx及修改配置
#cd /main/soft/
#tar -zxvf nginx-1.6.3.tar.gz
#cd nginx-1.6.3
#./configure --user=www --group=www --prefix=/main/server/nginx --with-http_stub_status_module --with-http_ssl_module --with-http_sub_module --with-pcre=../pcre-8.36 --with-pcre-jit --add-module=../fastdfs-nginx-module/src --add-module=../ngx_cache_purge-2.3
#make && make install
修改NGINX配置问文件:
# rm -rf /main/server/nginx/conf/nginx.conf
# vi /main/server/nginx/conf/nginx.conf
复制如下配置文件:
userwww www;
worker_processes4;
error_log/main/web/logs/nginx_error.logcrit;
#error_loglogs/error.log;
#error_loglogs/error.lognotice;
#error_loglogs/error.loginfo;
pid logs/nginx.pid;
events {
use epoll;
worker_connections 65535;
}
http{
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 300m;
include mime.types;
default_typeapplication/octet-stream;
sendfile on;
tcp_nopush on;
proxy_redirect off;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
proxy_buffer_size 16k;
proxy_buffers 4 64k;
proxy_busy_buffers_size 128k;
proxy_temp_file_write_size 128k;
keepalive_timeout65;
#设置缓存存储路径、存储方式、分配内存大小、磁盘最大空间、缓存期限
proxy_cache_path/main/web/cache/nginx/proxy_cachelevels=1:2keys_zone=http-cache:500m max_size=10g inactive=30d;
proxy_temp_path/main/web/cache/nginx/proxy_cache/tmp;
#设置group1的服务器
upstream fdfs_group1 {
server 192.168.2.192:8080 weight=1 max_fails=2 fail_timeout=30s;
server 192.168.2.193:8080 weight=1 max_fails=2 fail_timeout=30s;
}
#设置group2的服务器
upstream fdfs_group2 {
server 192.168.2.194:8080 weight=1 max_fails=2 fail_timeout=30s;
server 192.168.2.195:8080 weight=1 max_fails=2 fail_timeout=30s;
}
server {
listen 80;
server_namelocalhost;
#charset koi8-r;
#access_loglogs/host.access.logmain;
location /group1/M00{
proxy_next_upstream http_502 http_504 error timeout invalid_header;
proxy_cache_valid200 304 12h;
proxy_cache_key $uri$is_args$args;
proxy_pass http://fdfs_group1;
expires 30d;
}
location /group2/M00 {
proxy_next_upstream http_502 http_504 error timeout invalid_header;
proxy_cache_valid200 304 12h;
proxy_cache_key $uri$is_args$args;
proxy_pass http://fdfs_group2;
expires 30d;
}
location ~/purge(/.*) {
allow 127.0.0.1;
allow 192.168.2.0/24;
deny all;
}
}
log_formataccess'$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $http_x_forwarded_for';
access_log/main/web/logs/access.logaccess;
}
6、复制fastdfs相关文件到配置目录并修改
#cp /main/soft/fastdfs-5.05/conf/http.conf /etc/fdfs/
#cp /main/soft/fastdfs-5.05/conf/mime.types /etc/fdfs/
#cp /main/soft/fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/mod_fastdfs.conf
#vi /etc/fdfs/mod_fastdfs.conf
一般只需改动以下几个参数即可:
base_path=/main/web/fdfs/storage #保存日志目录
tracker_server=192.168.2.190:22122 #tracker服务器的IP地址以及端口号
tracker_server=192.168.2.191:22122 #tracker服务器的IP地址以及端口号
storage_server_port=23000 #storage服务器的端口号
group_name=group1 #当前服务器的group名
url_have_group_name= true #文件url中是否有group名
store_path_count=1 #存储路径个数,需要和store_path个数匹配
store_path0=/main/web/fdfs/storage #存储路径
group_count= 2 #设置组的个数
在末尾增加2个组的具体信息:
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/main/web/fdfs/storage
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/main/web/fdfs/storage
7、编写nginx 启动服务
#vi /etc/init.d/nginx
输入以下代码并保存
#!/bin/sh
#
# nginx - this script starts and stops the nginx daemin
#
# chkconfig:- 85 15
# description:Nginx is an HTTP(S) server, HTTP(S) reverse \
# proxy and IMAP/POP3 proxy server
# processname: nginx
# config: /main/server/nginx/conf/nginx.conf
# pidfile: /main/server/nginx/logs/nginx.pid
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "$NETWORKING" = "no" ] && exit 0
nginx="/main/server/nginx/sbin/nginx"
prog=$(basename $nginx)
NGINX_CONF_FILE="/main/server/nginx/conf/nginx.conf"
lockfile=/var/lock/subsys/nginx
start() {
[ -x $nginx ] || exit 5
[ -f $NGINX_CONF_FILE ] || exit 6
echo -n $"Starting $prog: "
daemon $nginx -c $NGINX_CONF_FILE
retval=$?
echo
[ $retval -eq 0 ] && touch $lockfile
return $retval
}
stop() {
echo -n $"Stopping $prog: "
killproc $prog -QUIT
retval=$?
echo
[ $retval -eq 0 ] && rm -f $lockfile
return $retval
}
restart() {
configtest || return $?
stop
start
}
reload() {
configtest || return $?
echo -n $"Reloading $prog: "
killproc $nginx -HUP
RETVAL=$?
echo
}
configtest() {
$nginx -t -c $NGINX_CONF_FILE
}
rh_status() {
status $prog
}
rh_status_q() {
rh_status >/dev/null 2>&1
}
case "$1" in
start)
rh_status_q && exit 0
$1
;;
stop)
rh_status_q || exit 0
$1
;;
restart|configtest)
$1
;;
reload)
rh_status_q || exit 7
$1
;;
status)
rh_status
;;
*)
echo $"Usage: $0 {start|stop|status|restart|reload|configtest}"
exit 2
esac
8、保存后,设置权限并添加到启动服务列表中 (2台tracker服务器配置)
#chmod 755 /etc/init.d/nginx
#chkconfig --add nginx
#chkconfig --level 345 nginx on
#service nginx start
第八部分tracker server两节点上配置client及测试
1、第一个节点修改
#cp /etc/fdfs/client.conf.sample /etc/fdfs/client.conf
#vi /etc/fdfs/client.conf
第一个及诶但修改以下参数:
base_path=/main/web/fdfs/tracker #日志存放路径
tracker_server=192.168.2.190:22122 #tracker服务器IP地址和端口号
http.tracker_server_port=8080 #tracker服务器的http端口号
2、第二个节点修改
#cp /etc/fdfs/client.conf.sample /etc/fdfs/client.conf
#vi /etc/fdfs/client.conf
第一个及诶但修改以下参数:
base_path=/main/web/fdfs/tracker #日志存放路径
tracker_server=192.168.2.191:22122 #tracker服务器IP地址和端口号
http.tracker_server_port=8080 #tracker服务器的http端口号
3、创建测试文件
#cd /main/
#vi test.html
hghghghg
上传测试文件
# /usr/bin/fdfs_test /etc/fdfs/client.conf upload test.html
第九部分tracker server两节点上安装keepalived实现nginx的高可用
1、keepalived安装
安装keepalived-1.2.16
#cd /main/soft
#tar zxvf keepalived-1.2.16.tar.gz
#cd keepalived-1.2.16
#./configure --prefix=/main/server/keepalived
#make && make install
复制相关目录
#复制keepalived启动文件到默认路径,也可以通过设置环境变量的path实现
#cp /main/server/keepalived/sbin/keepalived /usr/sbin/
#复制keepalived服务脚本到默认的地址,也通过修改init.d/keepalived文件中的相应配置实
#cp /main/server/keepalived/etc/sysconfig/keepalived /etc/sysconfig/
#复制服务启动脚本到,以便可以通过service控制keepalived服务
#cp /main/server/keepalived/etc/rc.d/init.d/keepalived /etc/init.d/
设置权限并添加到启动服务列表中
#chmod 755 /etc/init.d/keepalived
#chkconfig --add keepalived
#chkconfig --level 345 keepalived on
2、创建nginx监控脚本
#vi /main/server/nginx/conf/check_nginx.sh
##复制以下脚本
#!/bin/bash
A=`ps -C nginx –no-header|wc -l`
if [ $A -eq 0 ];then
/main/server/nginx/sbin/nginx
sleep 3
if [ `ps -C nginx --no-header|wc -l` -eq 0 ];then
killall keepalived
fi
fi
3、给nginx监控脚本赋予权限
#chmod 755 /main/server/nginx/conf/check_nginx.sh
4、编写配置文件
主节点(10.1.11.31)创建keepalived配置文件
#vi /etc/keepalived/keepalived.conf
##复制以下配置文件
!Configuration File for keepalived
global_defs {
notification_email {
jk@hgtech365.com
}
notification_email_from jk@hgtech365.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX1_DEVEL
}
vrrp_script chk_nginx {
script "/main/server/nginx/conf/check_nginx.sh"
interval 2
weight 2
}
vrrp_instance VI_1 {
state MASTER
nopreempt
interface eth0
virtual_router_id 31
mcast_src_ip 192.168.2.190
priority 150
advert_int 1
authentication {
auth_type PASS
auth_pass fds#FSAF897
}
virtual_ipaddress {
192.168.2.196
}
track_script {
chk_nginx
}
}
5、备用节点(192.168.2.191)创建keepalived配置文件
#vi /etc/keepalived/keepalived.conf
##复制以下配置文件
! Configuration File for keepalived
global_defs {
notification_email {
jk@hgtech365.com
}
notification_email_from flb@flb41.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX2_DEVEL
}
vrrp_script chk_nginx {
script "/main/server/nginx/conf/check_nginx.sh"
interval 2
weight 2
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 31
mcast_src_ip 192.168.2.191
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass fds#FSAF897
}
virtual_ipaddress {
192.168.2.196
}
track_script {
chk_nginx
}
}
5、保存配置文件后,启动keepalived服务
#service keepalived start
6、防火墙设置
端口开放
#iptables -I INPUT -p tcp --dport 80 -j ACCEPT
#iptables -I INPUT 3 -p vrrp -j ACCEPT
#/etc/rc.d/init.d/iptables save
6、测试
http://192.168.100.240
页:
[1]