|
|
环境：centos6.5
tracker_server_1 ip：192.168.2.190 tracker_server_2 ip：192.168.2.191
storage_server_group1_1 ip: 192.168.2.192 storage_server_group1_2 ip:192.168.2.193
storage_server_group2_1 ip: 192.168.2.194 storage_server_group2_2 ip:192.168.2.195
第一部分六节点做时间同步
#service iptables stop
# yum install -y ntpdate
# cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
# ntpdate us.pool.ntp.org
时间定时同步(6节点安装)
# crontab -e
输入以下代码，每10分钟同步一次
*/10 * * * * /usr/sbin/ntpdate us.pool.ntp.org | logger -t NTP
第二部分六节点安装FastDFS_v5.05
1、安装相关倚赖包并建立相关文件夹及权限设定
# mkdir /main/soft
所有相关软件上传至/main/soft/
# yum -y install gcc gcc-c++ openssl openssl-devel pcre-devel perl unzip
# /usr/sbin/groupadd www
# /usr/sbin/useradd -s /sbin/nologin -M -g www www
# mkdir -p /main/web/fdfs
# chmod 777 /main/web/fdfs
# mkdir /main/web/fdfs/storage
# chown -R www:www /main/web/fdfs
# chmod g+s /main/web/fdfs
# mkdir -p /main/web/logs
# chmod +w /main/web/logs
# chown -R www:www /main/web/logs
2、安装libfastcommon，FastDFS_v5.05依赖libfastcommon，不再依赖libevent
# cd /main/soft/
# unzip libfastcommon-master.zip
# cd libfastcommon-master
# ./make.sh
# ./make.sh install
libfastcommon.so默认安装到了/usr/lib64/libfastcommon.so，而FastDFS主程序设置的lib目录是/usr/local/lib，所以设置软连接
# ln -s /usr/lib64/libfastcommon.so /usr/local/lib/libfastcommon.so
# ln -s /usr/lib64/libfastcommon.so /usr/lib/libfastcommon.so
# ln -s /usr/lib64/libfdfsclient.so /usr/local/lib/libfdfsclient.so
# ln -s /usr/lib64/libfdfsclient.so /usr/lib/libfdfsclient.so
3、安装fastdfs
# cd /main/soft
# tar -zxvf fastdfs-5.05.tar.gz
# cd fastdfs-5.05/
# ./make.sh
# ./make.sh install
第三部分二个tracker节点配置
1、建立数据及日志目录
# mkdir /main/web/fdfs/tracker
2、拷贝及修改tracker配置文件
# cp /etc/fdfs/tracker.conf.sample /etc/fdfs/tracker.conf
# vi /etc/fdfs/tracker.conf
一般只需改动以下几个参数即可：
disabled=false #启用配置文件
port=22122 #设置tracker的端口号
base_path=/main/web/fdfs/tracker #设置tracker的数据文件和日志目录（需预先创建）
http.server_port=8080 #设置http端口号
3、启动并验证tracker节点
# /usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf
# echo '/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf' >> /etc/rc.local
启动tracker，确认启动是否成功。（查看是否对应端口22122是否开始监听）
# netstat -unltp|grep fdfs
也可查看tracker的日志是否启动成功或是否有错误。
# cat /main/web/fdfs/tracker/logs/trackerd.log
第四部分storage四节点配置
1、防火墙设置
端口开放
# iptables -I INPUT -p tcp --dport 8080 -j ACCEPT
# iptables -I INPUT -p tcp --dport 22122 -j ACCEPT
# /etc/rc.d/init.d/iptables save
2、拷贝及修改storage配置文件
# cp /etc/fdfs/storage.conf.sample /etc/fdfs/storage.conf
# vi /etc/fdfs/storage.conf
一般只需改动以下几个参数即可：
disabled=false #启用配置文件
group_name=group1 #组名，根据实际情况修改
port=23000 #设置storage的端口号
base_path=/main/web/fdfs/storage #设置storage的日志目录（需预先创建）
store_path_count=1 #存储路径个数，需要和store_path个数匹配
store_path0=/main/web/fdfs/storage #存储路径
tracker_server=192.168.2.190:22122 #tracker服务器的IP地址和端口号
tracker_server=192.168.2.191:22122 #tracker服务器的IP地址和端口号
http.server_port=8080 #设置http端口号
3、启动storage服务
# /usr/bin/fdfs_storaged /etc/fdfs/storage.conf
重启storage服务
# /usr/bin/restart.sh /usr/bin/fdfs_storaged /etc/fdfs/storage.conf
# cat /main/web/fdfs/storage/logs/storaged.log
确认启动成功后，可以运行fdfs_monitor查看storage服务器是否已经登记到tracker服务器，特别注意每个group中所有storage的端口号必须一致
# /usr/bin/fdfs_monitor /etc/fdfs/storage.conf
第五部分storage四节点上安装NGINX，在storage上安装的nginx主要为了提供http的访问服务，同时解决group中storage服务器的同步延迟问题。(需要安装ngx_cache_purge-2.3.tar.gz)
1、在storage server安装nginx模块
# cd /main/soft/
# tar -zxvf fastdfs-nginx-module_v1.16.tar.gz
# vi /main/soft/fastdfs-nginx-module/src/config
修改如下配置，我这里原来是
CORE_INCS="$CORE_INCS /usr/local/include/fastdfs /usr/local/include/fastcommon/"
改成
CORE_INCS="$CORE_INCS /usr/include/fastdfs /usr/include/fastcommon/"
这个是很重要的，不然在nginx编译的时候会报错的，我看网上很多在安装nginx的fastdfs的插件报错，都是这个原因，而不是版本不匹配。
2、安装Nginx所需的pcre库
# cd /main/soft
# tar zxvf pcre-8.36.tar.gz
# cd pcre-8.36/
# ./configure
# make && make install
3、安装nginx及修改配置
# cd /main/soft/
# tar -zxvf nginx-1.6.3.tar.gz
# cd nginx-1.6.3
# ./configure --user=www --group=www --prefix=/main/server/nginx --with-http_stub_status_module --with-http_ssl_module --with-http_sub_module --with-pcre=../pcre-8.36 --with-pcre-jit --add-module=../fastdfs-nginx-module/src
# make && make install
4、修改nginx配置
# rm -rf /main/server/nginx/conf/nginx.conf
# vi /main/server/nginx/conf/nginx.conf
复制以下代码
user www www;
worker_processes 4;
error_log /main/web/logs/nginx_error.log crit;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
pid logs/nginx.pid;
events {
use epoll;
worker_connections 65535;
}
http{
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 300m;
include mime.types;
default_type application/octet-stream;
sendfile on;
tcp_nopush on;
proxy_redirect off;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
proxy_buffer_size 16k;
proxy_buffers 4 64k;
proxy_busy_buffers_size 128k;
proxy_temp_file_write_size 128k;
keepalive_timeout 65;
server {
listen 8080;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
location ~/group[1-2]/M00{
root /main/web/fdfs/storage/data;
ngx_fastdfs_module;
}
}
log_format access '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $http_x_forwarded_for';
access_log /main/web/logs/access.log access;
}
5、复制fastdfs相关文件到配置目录并修改
# cp /main/soft/fastdfs-5.05/conf/http.conf /etc/fdfs/
# cp /main/soft/fastdfs-5.05/conf/mime.types /etc/fdfs/
# cp /main/soft/fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/mod_fastdfs.conf
# vi /etc/fdfs/mod_fastdfs.conf
一般只需改动以下几个参数即可：
base_path=/main/web/fdfs/storage #保存日志目录
tracker_server=192.168.2.190:22122 #tracker服务器的IP地址以及端口号
tracker_server=192.168.2.191:22122 #tracker服务器的IP地址以及端口号
storage_server_port=23000 #storage服务器的端口号
group_name=group1 #当前服务器的group名
url_have_group_name= true #文件url中是否有group名
store_path_count=1 #存储路径个数，需要和store_path个数匹配
store_path0=/main/web/fdfs/storage #存储路径
group_count= 2 #设置组的个数
在末尾增加2个组的具体信息：
[group1]
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/main/web/fdfs/storage
[group2]
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/main/web/fdfs/storage
6、给storage的存储目录做一个软连接
# ln -s /main/web/fdfs/storage/data /main/web/fdfs/storage/data/M00
7、编写nginx 启动服务
# vi /etc/init.d/nginx
输入以下代码并保存（4台storage服务器配置）
#!/bin/sh
#
# nginx - this script starts and stops the nginx daemin
#
# chkconfig: - 85 15
# description: Nginx is an HTTP(S) server, HTTP(S) reverse \
# proxy and IMAP/POP3 proxy server
# processname: nginx
# config: /main/server/nginx/conf/nginx.conf
# pidfile: /main/server/nginx/logs/nginx.pid
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "$NETWORKING" = "no" ] && exit 0
nginx="/main/server/nginx/sbin/nginx"
prog=$(basename $nginx)
NGINX_CONF_FILE="/main/server/nginx/conf/nginx.conf"
lockfile=/var/lock/subsys/nginx
start() {
[ -x $nginx ] || exit 5
[ -f $NGINX_CONF_FILE ] || exit 6
echo -n $"Starting $prog: "
daemon $nginx -c $NGINX_CONF_FILE
retval=$?
echo
[ $retval -eq 0 ] && touch $lockfile
return $retval
}
stop() {
echo -n $"Stopping $prog: "
killproc $prog -QUIT
retval=$?
echo
[ $retval -eq 0 ] && rm -f $lockfile
return $retval
}
restart() {
configtest || return $?
stop
start
}
reload() {
configtest || return $?
echo -n $"Reloading $prog: "
killproc $nginx -HUP
RETVAL=$?
echo
}
configtest() {
$nginx -t -c $NGINX_CONF_FILE
}
rh_status() {
status $prog
}
rh_status_q() {
rh_status >/dev/null 2>&1
}
case "$1" in
start)
rh_status_q && exit 0
$1
;;
stop)
rh_status_q || exit 0
$1
;;
restart|configtest)
$1
;;
reload)
rh_status_q || exit 7
$1
;;
status)
rh_status
;;
*)
echo $"Usage: $0 {start|stop|status|restart|reload|configtest}"
exit 2
esac
保存后，设置权限并添加到启动服务列表中
# chmod 755 /etc/init.d/nginx
# chkconfig --add nginx
# chkconfig --level 345 nginx on
# service nginx start
在error.log中没有错误，既启动成功。可以打开浏览器，直接访问http://192.168.2.192:8080，查看是否弹出nginx欢迎页面。
第六部分tracker两节点上安装nginx，在tracker上安装的nginx主要为了提供http访问的反向代理、负载均衡以及缓存服务。
1、建立相关文件夹
# mkdir -p /main/web/cache/nginx/proxy_cache/tmp
# chown -R www:www /main/web/cache
# chmod g+s /main/web/cache
# mkdir /etc/keepalived
2、解压fastdfs-nginx-module模块
# cd /main/soft/
# tar -zxvf fastdfs-nginx-module_v1.16.tar.gz
# vi /main/soft/fastdfs-nginx-module/src/config
修改如下配置，我这里原来是
CORE_INCS="$CORE_INCS /usr/local/include/fastdfs /usr/local/include/fastcommon/"
改成
CORE_INCS="$CORE_INCS /usr/include/fastdfs /usr/include/fastcommon/"
这个是很重要的，不然在nginx编译的时候会报错的，我看网上很多在安装nginx的fastdfs的插件报错，都是这个原因，而不是版本不匹配。
3、解压ngx_cache_purge模块
# tar zxvf ngx_cache_purge-2.3.tar.gz
4、安装Nginx所需的pcre库
# tar zxvf pcre-8.36.tar.gz
# cd pcre-8.36/
# ./configure
# make && make install
5、安装nginx及修改配置
# cd /main/soft/
# tar -zxvf nginx-1.6.3.tar.gz
# cd nginx-1.6.3
# ./configure --user=www --group=www --prefix=/main/server/nginx --with-http_stub_status_module --with-http_ssl_module --with-http_sub_module --with-pcre=../pcre-8.36 --with-pcre-jit --add-module=../fastdfs-nginx-module/src --add-module=../ngx_cache_purge-2.3
# make && make install
修改NGINX配置问文件：
# rm -rf /main/server/nginx/conf/nginx.conf
# vi /main/server/nginx/conf/nginx.conf
复制如下配置文件：
user www www;
worker_processes 4;
error_log /main/web/logs/nginx_error.log crit;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
pid logs/nginx.pid;
events {
use epoll;
worker_connections 65535;
}
http{
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 300m;
include mime.types;
default_type application/octet-stream;
sendfile on;
tcp_nopush on;
proxy_redirect off;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
proxy_buffer_size 16k;
proxy_buffers 4 64k;
proxy_busy_buffers_size 128k;
proxy_temp_file_write_size 128k;
keepalive_timeout 65;
#设置缓存存储路径、存储方式、分配内存大小、磁盘最大空间、缓存期限
proxy_cache_path /main/web/cache/nginx/proxy_cache levels=1:2 keys_zone=http-cache:500m max_size=10g inactive=30d;
proxy_temp_path /main/web/cache/nginx/proxy_cache/tmp;
#设置group1的服务器
upstream fdfs_group1 {
server 192.168.2.192:8080 weight=1 max_fails=2 fail_timeout=30s;
server 192.168.2.193:8080 weight=1 max_fails=2 fail_timeout=30s;
}
#设置group2的服务器
upstream fdfs_group2 {
server 192.168.2.194:8080 weight=1 max_fails=2 fail_timeout=30s;
server 192.168.2.195:8080 weight=1 max_fails=2 fail_timeout=30s;
}
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
location /group1/M00{
proxy_next_upstream http_502 http_504 error timeout invalid_header;
proxy_cache_valid 200 304 12h;
proxy_cache_key $uri$is_args$args;
proxy_pass http://fdfs_group1;
expires 30d;
}
location /group2/M00 {
proxy_next_upstream http_502 http_504 error timeout invalid_header;
proxy_cache_valid 200 304 12h;
proxy_cache_key $uri$is_args$args;
proxy_pass http://fdfs_group2;
expires 30d;
}
location ~/purge(/.*) {
allow 127.0.0.1;
allow 192.168.2.0/24;
deny all;
}
}
log_format access '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $http_x_forwarded_for';
access_log /main/web/logs/access.log access;
}
6、复制fastdfs相关文件到配置目录并修改
# cp /main/soft/fastdfs-5.05/conf/http.conf /etc/fdfs/
# cp /main/soft/fastdfs-5.05/conf/mime.types /etc/fdfs/
# cp /main/soft/fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/mod_fastdfs.conf
# vi /etc/fdfs/mod_fastdfs.conf
一般只需改动以下几个参数即可：
base_path=/main/web/fdfs/storage #保存日志目录
tracker_server=192.168.2.190:22122 #tracker服务器的IP地址以及端口号
tracker_server=192.168.2.191:22122 #tracker服务器的IP地址以及端口号
storage_server_port=23000 #storage服务器的端口号
group_name=group1 #当前服务器的group名
url_have_group_name= true #文件url中是否有group名
store_path_count=1 #存储路径个数，需要和store_path个数匹配
store_path0=/main/web/fdfs/storage #存储路径
group_count= 2 #设置组的个数
在末尾增加2个组的具体信息：
[group1]
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/main/web/fdfs/storage
[group2]
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/main/web/fdfs/storage
7、编写nginx 启动服务
# vi /etc/init.d/nginx
输入以下代码并保存
#!/bin/sh
#
# nginx - this script starts and stops the nginx daemin
#
# chkconfig: - 85 15
# description: Nginx is an HTTP(S) server, HTTP(S) reverse \
# proxy and IMAP/POP3 proxy server
# processname: nginx
# config: /main/server/nginx/conf/nginx.conf
# pidfile: /main/server/nginx/logs/nginx.pid
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "$NETWORKING" = "no" ] && exit 0
nginx="/main/server/nginx/sbin/nginx"
prog=$(basename $nginx)
NGINX_CONF_FILE="/main/server/nginx/conf/nginx.conf"
lockfile=/var/lock/subsys/nginx
start() {
[ -x $nginx ] || exit 5
[ -f $NGINX_CONF_FILE ] || exit 6
echo -n $"Starting $prog: "
daemon $nginx -c $NGINX_CONF_FILE
retval=$?
echo
[ $retval -eq 0 ] && touch $lockfile
return $retval
}
stop() {
echo -n $"Stopping $prog: "
killproc $prog -QUIT
retval=$?
echo
[ $retval -eq 0 ] && rm -f $lockfile
return $retval
}
restart() {
configtest || return $?
stop
start
}
reload() {
configtest || return $?
echo -n $"Reloading $prog: "
killproc $nginx -HUP
RETVAL=$?
echo
}
configtest() {
$nginx -t -c $NGINX_CONF_FILE
}
rh_status() {
status $prog
}
rh_status_q() {
rh_status >/dev/null 2>&1
}
case "$1" in
start)
rh_status_q && exit 0
$1
;;
stop)
rh_status_q || exit 0
$1
;;
restart|configtest)
$1
;;
reload)
rh_status_q || exit 7
$1
;;
status)
rh_status
;;
*)
echo $"Usage: $0 {start|stop|status|restart|reload|configtest}"
exit 2
esac
8、保存后，设置权限并添加到启动服务列表中 （2台tracker服务器配置）
# chmod 755 /etc/init.d/nginx
# chkconfig --add nginx
# chkconfig --level 345 nginx on
# service nginx start
第八部分tracker server两节点上配置client及测试
1、第一个节点修改
# cp /etc/fdfs/client.conf.sample /etc/fdfs/client.conf
# vi /etc/fdfs/client.conf
第一个及诶但修改以下参数：
base_path=/main/web/fdfs/tracker #日志存放路径
tracker_server=192.168.2.190:22122 #tracker服务器IP地址和端口号
http.tracker_server_port=8080 #tracker服务器的http端口号
2、第二个节点修改
# cp /etc/fdfs/client.conf.sample /etc/fdfs/client.conf
# vi /etc/fdfs/client.conf
第一个及诶但修改以下参数：
base_path=/main/web/fdfs/tracker #日志存放路径
tracker_server=192.168.2.191:22122 #tracker服务器IP地址和端口号
http.tracker_server_port=8080 #tracker服务器的http端口号
3、创建测试文件
# cd /main/
# vi test.html
hghghghg
上传测试文件
# /usr/bin/fdfs_test /etc/fdfs/client.conf upload test.html
第九部分tracker server两节点上安装keepalived实现nginx的高可用
1、keepalived安装
安装keepalived-1.2.16
# cd /main/soft
# tar zxvf keepalived-1.2.16.tar.gz
# cd keepalived-1.2.16
# ./configure --prefix=/main/server/keepalived
# make && make install
复制相关目录
#复制keepalived启动文件到默认路径，也可以通过设置环境变量的path实现
# cp /main/server/keepalived/sbin/keepalived /usr/sbin/
#复制keepalived服务脚本到默认的地址，也通过修改init.d/keepalived文件中的相应配置实
# cp /main/server/keepalived/etc/sysconfig/keepalived /etc/sysconfig/
#复制服务启动脚本到，以便可以通过service控制keepalived服务
# cp /main/server/keepalived/etc/rc.d/init.d/keepalived /etc/init.d/
设置权限并添加到启动服务列表中
# chmod 755 /etc/init.d/keepalived
# chkconfig --add keepalived
# chkconfig --level 345 keepalived on
2、创建nginx监控脚本
# vi /main/server/nginx/conf/check_nginx.sh
##复制以下脚本
#!/bin/bash
A=`ps -C nginx –no-header|wc -l`
if [ $A -eq 0 ];then
/main/server/nginx/sbin/nginx
sleep 3
if [ `ps -C nginx --no-header|wc -l` -eq 0 ];then
killall keepalived
fi
fi
3、给nginx监控脚本赋予权限
# chmod 755 /main/server/nginx/conf/check_nginx.sh
4、编写配置文件
主节点（10.1.11.31）创建keepalived配置文件
# vi /etc/keepalived/keepalived.conf
##复制以下配置文件
!Configuration File for keepalived
global_defs {
notification_email {
jk@hgtech365.com
}
notification_email_from jk@hgtech365.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX1_DEVEL
}
vrrp_script chk_nginx {
script "/main/server/nginx/conf/check_nginx.sh"
interval 2
weight 2
}
vrrp_instance VI_1 {
state MASTER
nopreempt
interface eth0
virtual_router_id 31
mcast_src_ip 192.168.2.190
priority 150
advert_int 1
authentication {
auth_type PASS
auth_pass fds#FSAF897
}
virtual_ipaddress {
192.168.2.196
}
track_script {
chk_nginx
}
}
5、备用节点（192.168.2.191）创建keepalived配置文件
# vi /etc/keepalived/keepalived.conf
##复制以下配置文件
! Configuration File for keepalived
global_defs {
notification_email {
jk@hgtech365.com
}
notification_email_from flb@flb41.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX2_DEVEL
}
vrrp_script chk_nginx {
script "/main/server/nginx/conf/check_nginx.sh"
interval 2
weight 2
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 31
mcast_src_ip 192.168.2.191
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass fds#FSAF897
}
virtual_ipaddress {
192.168.2.196
}
track_script {
chk_nginx
}
}
5、保存配置文件后，启动keepalived服务
# service keepalived start
6、防火墙设置
端口开放
# iptables -I INPUT -p tcp --dport 80 -j ACCEPT
# iptables -I INPUT 3 -p vrrp -j ACCEPT
# /etc/rc.d/init.d/iptables save
6、测试
http://192.168.100.240
|
|
|
|
|
|
|
|
|
QQ群⑧：
窥视卡
雷达卡
发表于 2019-1-31 13:45:06
提升卡
置顶卡
沉默卡
喧嚣卡
变色卡
千斤顶
显身卡