FastDFS之文件服务器集群部署详解
FastDFS是一个开源的轻量级分布式文件系统，由跟踪服务器（tracker server）、存储服务器（storage server）和客户端（client）三个部分组成，主要解决了海量数据存储问题，特别适合以中小文件（建议范围：4KB < file_size <500MB）为载体的在线服务。在生成环境FastDFS一般都是用集群配置，以提高FastDFS的可用性，并发能力。部署架构：
环境IP地址(关闭所有环境的防火墙)：
Tracker 192.168.18.178
Group 1：
S1：192.168.110.71
S2：192.168.110.91
Group 2：
S3：192.168.100.90
S4：192.168.100.194
注：Tracker可以部署多台，提供负载，这里资源有限，就部署一台。
由于需要安装nginx，每台机器都安装依赖：
yum -y installzlib pcre pcre-devel zlib-devel
所用的安装软件下载：http://download.csdn.net/detail/tianwei7518/8745279
一、安装tracker
1.安装依赖libfastcommon
unzip libfastcommon-master.zip
cd libfastcommon
./make.sh
./make.shinstall
2.安装FastDFS
unzip fastdfs.zip
cd fastdfs
./make.sh
./make.sh install
默认安装目录：/usr/bin
将原安装文件夹下的配置文件复制到/etc/fdfs目下：cp ./conf/*/etc/fdfs/
3.配置
编辑配置文件目录下的tracker.conf
一般只需改动以下几个参数即可：
disabled=false #启用配置文件
port=22122 #设置tracker的端口号
base_path=/home/fastdfs #设置tracker的数据文件和日志目录（需预先创建）
http.server_port=8080 #设置http端口号
4.启动
/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf start
二、安装tracker代理nginx
在tracker上安装的nginx主要为了提供http访问的反向代理、负载均衡以及缓存服务。1.安装nginxtar -zxvf nginx-1.8.0.tar.gz
tar -zxvf ngx_cache_purge-2.3.tar.gz
cd nginx-1.8.0
./configure --prefix=/usr/local/nginx --add-module=/root/ngx_cache_purge-2.3
make
make install
如果提示错误，可能缺少依赖的软件包，需先安装依赖包，再次运行./configure
nginx以及nginx cache purge插件模块安装完成，安装目录/usr/local/nginx
2.配置nginx
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
userroot;
worker_processes1;
#error_loglogs/error.log;
#error_loglogs/error.lognotice;
error_loglogs/error.loginfo;
#pid logs/nginx.pid;
events {
worker_connections1024;
}
http {
include mime.types;
default_typeapplication/octet-stream;
#设置缓存参数
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 300m;
sendfile on;
tcp_nopush on;
proxy_redirect off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
proxy_buffer_size 16k;
proxy_buffers 4 64k;
proxy_busy_buffers_size 128k;
proxy_temp_file_write_size 128k;
#设置缓存存储路径、存储方式、分配内存大小、磁盘最大空间、缓存期限
proxy_cache_path /var/cache/nginx/proxy_cache levels=1:2 keys_zone=http-cache:500m max_size=10g inactive=30d;
proxy_temp_path /var/cache/nginx/proxy_cache/tmp;
keepalive_timeout65;
#设置group服务器
upstream fdfs_group1 {
server 192.168.110.71:8090 weight=1 max_fails=2 fail_timeout=30s;
server 192.168.110.91:8090 weight=1 max_fails=2 fail_timeout=30s;
}
upstream fdfs_group2 {
server 192.168.100.90:8090 weight=1 max_fails=2 fail_timeout=30s;
server 192.168.100.194:8090 weight=1 max_fails=2 fail_timeout=30s;
}
server {
listen 80;
server_namelocalhost;
charset utf-8;
#access_log/usr/local/nginx/logs/localhost.access.logmain;
location /group1/M00 {
proxy_next_upstream http_502 http_504 error timeout invalid_header;
proxy_cache http-cache;
proxy_cache_valid200 304 12h;
proxy_cache_key $uri$is_args$args;
proxy_pass http://fdfs_group1;
expires 30d;
}
location /group2/M00 {
proxy_next_upstream http_502 http_504 error timeout invalid_header;
proxy_cache http-cache;
proxy_cache_valid200 304 12h;
proxy_cache_key $uri$is_args$args;
proxy_pass http://fdfs_group2;
expires 30d;
}
#设置清除缓存的访问权限
location ~ /purge(/.*) {
allow 127.0.0.1;
allow 172.16.1.0/24;
deny all;
proxy_cache_purge http-cache$1$is_args$args;
}
}
}
创建缓存目录：/var/cache/nginx/proxy_cache/tmp3.启动
/usr/local/nginx/sbin/nginx
三、安装storage
1.安装
参考安装tracker前2步骤。
2.配置
编辑配置文件目录下的storage.conf
只需改动以下几个参数即可：
disabled=false #启用配置文件
group_name=group1 #组名，根据实际情况修改
port=23000 #设置storage的端口号
base_path=/home/fastdfs #设置storage的日志目录（需预先创建）
store_path_count=1 #存储路径个数，需要和store_path个数匹配
store_path0=/home/fastdfs#存储路径
tracker_server=192.168.18.178:22122#tracker服务器的IP地址和端口号
http.server_port=8080 #设置http端口号
创建目录mkdir /home/fastdfs
3.运行
/usr/bin/fdfs_storaged /etc/fdfs/storage.conf start
另外：
分别在其他机器上全部安装storage并确认运行正常。注意配置文件中group名参数需要根据实际情况调整：
group1：192.168.110.71，192.168.110.91
group2：192.168.100.90，192.168.100.194
另外每个group中所有storage的端口号必须一致。
四、在storage上安装nginx
在storage上安装的nginx主要为了提供http的访问服务，同时解决group中storage服务器的同步延迟问题。
1.解压fastdfs-nginx-module插件
unzipfastdfs-nginx-module.zip
2.安装nginx
tar -zxvf nginx-1.8.0.tar.gz
cd nginx-1.8.0
./configure --prefix=/usr/local/nginx --add-module=../fastdfs-nginx-module/src
make
make install
安装目录：/usr/local/nginx
若安装报错： 13513#0: eventfd() failed (38: Function not implemented)
原因是：编译时带了--with-file-aio模块，这个要Linux 2.6.22以后内核才支持．服务器是2.6.18。也可以下载低版本的nginx版本
3.配置
1）配置FastDFS的nginx插件
将FastDFS的nginx插件模块的配置文件copy到FastDFS配置文件目录
fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/
编辑/etc/fdfs配置文件目录下的mod_fastdfs.conf，设置storage信息并保存。
一般只需改动以下几个参数即可：
base_path=/home/fastdfs #保存日志目录
tracker_server=192.168.18.178:22122 #tracker服务器的IP地址以及端口号
storage_server_port=23000#storage服务器的端口号
group_name=group1#当前服务器的group名
url_have_group_name = true #文件url中是否有group名
store_path_count=1 #存储路径个数，需要和store_path个数匹配
store_path0=/home/fastdfs #存储路径
http.need_find_content_type=true#从文件扩展名查找文件类型（nginx时为true）
group_count = 2 #设置组的个数
在末尾增加2个组的具体信息：
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/home/fastdfs
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/home/fastdfs
建立M00至存储目录的符号连接：
ln -s /home/fastdfs/data /home/fastdfs/data/M00
2）配置nginx
vi nginx.conf
user root;
location ~/group/M00 {
root /home/fastdfs/data;
ngx_fastdfs_module;
}
4.启动
/usr/local/nginx/sbin/nginx
另外：
分别在其他机器storage上全部安装nginx并确认运行正常。注意配置文件中group名参数需要根据实际情况调整：
group1：192.168.110.71，192.168.110.91
group2：192.168.100.90，192.168.100.194
另外nginx的端口号8090。
至此所有配置完毕。
四、测试
配置/etc/fdfs/client.conf
base_path=/home/fastdfs #日志存放路径
tracker_server=192.168.18.43:22122 #tracker服务器IP地址和端口号
http.tracker_server_port=8080 #tracker服务器的http端口号
通过fdfs_upload_file上传一个文件到FastDFS，程序会自动返回文件的URL，
#fdfs_upload_file /etc/fdfs/client.conf 40-15052PZK5.jpg
group1/M00/00/00/wKhuR1Vmh_2ADmdfAAF1dmVtk4w934.jpg
然后使用浏览器访问，访问正常
http://192.168.18.43/group1/M00/00/00/wKhuR1Vmh_2ADmdfAAF1dmVtk4w934.jpg
注：可以使用fdfs_monitor查看tracker和所有group的运行情况
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
# fdfs_monitor /etc/fdfs/client.conf
DEBUG - base_path=/home/fastdfs, connect_timeout=30, network_timeout=60, tracker_server_count=1, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0
server_count=1, server_index=0
tracker server is 192.168.18.43:22122
group count: 2
Group 1:
group name = group1
disk total space = 45438 MB
disk free space = 33920 MB
trunk free space = 0 MB
storage server count = 2
active server count = 2
storage server port = 23000
storage HTTP port = 8080
store path count = 1
subdir count per path = 256
current write server index = 1
current trunk file id = 0
Storage 1:
id = 192.168.110.71
ip_addr = 192.168.110.71 (localhost)ACTIVE
http domain =
version = 5.06
join time = 2015-05-27 01:37:04
up time =
total storage = 95217 MB
free storage = 47563 MB
upload priority = 10
store_path_count = 1
subdir_count_per_path = 256
storage_port = 23000
storage_http_port = 8080
current_write_path = 0
source storage id =
if_trunk_server = 0
connection.alloc_count = 256
connection.current_count = 1
connection.max_count = 2
total_upload_count = 1
success_upload_count = 1
total_append_count = 0
success_append_count = 0
total_modify_count = 0
success_modify_count = 0
total_truncate_count = 0
success_truncate_count = 0
total_set_meta_count = 0
success_set_meta_count = 0
total_delete_count = 0
success_delete_count = 0
total_download_count = 0
success_download_count = 0
total_get_meta_count = 0
success_get_meta_count = 0
total_create_link_count = 0
success_create_link_count = 0
total_delete_link_count = 0
success_delete_link_count = 0
total_upload_bytes = 95606
success_upload_bytes = 95606
total_append_bytes = 0
success_append_bytes = 0
total_modify_bytes = 0
success_modify_bytes = 0
stotal_download_bytes = 0
success_download_bytes = 0
total_sync_in_bytes = 0
success_sync_in_bytes = 0
total_sync_out_bytes = 0
success_sync_out_bytes = 0
total_file_open_count = 1
success_file_open_count = 1
total_file_read_count = 0
success_file_read_count = 0
total_file_write_count = 1
success_file_write_count = 1
last_heart_beat_time = 2015-05-27 20:19:49
last_source_update = 2015-05-27 20:14:04
last_sync_update = 1969-12-31 16:00:00
last_synced_timestamp = 1969-12-31 16:00:00
Storage 2:
id = 192.168.110.91
ip_addr = 192.168.110.91 (localhost)ACTIVE
http domain =
version = 5.06
join time = 2015-05-27 19:35:36
up time = 2015-05-27 19:35:36
total storage = 45438 MB
free storage = 33920 MB
upload priority = 10
store_path_count = 1
subdir_count_per_path = 256
storage_port = 23000
storage_http_port = 8080
current_write_path = 0
source storage id = 192.168.110.71
if_trunk_server = 0
connection.alloc_count = 256
connection.current_count = 1
connection.max_count = 1
total_upload_count = 0
success_upload_count = 0
total_append_count = 0
success_append_count = 0
total_modify_count = 0
success_modify_count = 0
total_truncate_count = 0
success_truncate_count = 0
total_set_meta_count = 0
success_set_meta_count = 0
total_delete_count = 0
success_delete_count = 0
total_download_count = 0
success_download_count = 0
total_get_meta_count = 0
success_get_meta_count = 0
total_create_link_count = 0
success_create_link_count = 0
total_delete_link_count = 0
success_delete_link_count = 0
total_upload_bytes = 0
success_upload_bytes = 0
total_append_bytes = 0
success_append_bytes = 0
total_modify_bytes = 0
success_modify_bytes = 0
stotal_download_bytes = 0
success_download_bytes = 0
total_sync_in_bytes = 95606
success_sync_in_bytes = 95606
total_sync_out_bytes = 0
success_sync_out_bytes = 0
total_file_open_count = 1
success_file_open_count = 1
total_file_read_count = 0
success_file_read_count = 0
total_file_write_count = 1
success_file_write_count = 1
last_heart_beat_time = 2015-05-27 20:19:51
last_source_update = 1969-12-31 16:00:00
last_sync_update = 2015-05-27 20:14:07
last_synced_timestamp = 2015-05-27 20:14:05 (-1s delay)
Group 2:
group name = group2
disk total space = 9916 MB
disk free space = 7434 MB
trunk free space = 0 MB
storage server count = 2
active server count = 2
storage server port = 23000
storage HTTP port = 8080
store path count = 1
subdir count per path = 256
current write server index = 0
current trunk file id = 0
Storage 1:
id = 192.168.100.194
ip_addr = 192.168.100.194 (localhost)ACTIVE
http domain =
version = 5.06
join time = 2015-05-27 20:03:37
up time = 2015-05-27 20:03:37
total storage = 47368 MB
free storage = 37371 MB
upload priority = 10
store_path_count = 1
subdir_count_per_path = 256
storage_port = 23000
storage_http_port = 8080
current_write_path = 0
source storage id = 192.168.100.90
if_trunk_server = 0
connection.alloc_count = 256
connection.current_count = 1
connection.max_count = 1
total_upload_count = 0
success_upload_count = 0
total_append_count = 0
success_append_count = 0
total_modify_count = 0
success_modify_count = 0
total_truncate_count = 0
success_truncate_count = 0
total_set_meta_count = 0
success_set_meta_count = 0
total_delete_count = 0
success_delete_count = 0
total_download_count = 0
success_download_count = 0
total_get_meta_count = 0
success_get_meta_count = 0
total_create_link_count = 0
success_create_link_count = 0
total_delete_link_count = 0
success_delete_link_count = 0
total_upload_bytes = 0
success_upload_bytes = 0
total_append_bytes = 0
success_append_bytes = 0
total_modify_bytes = 0
success_modify_bytes = 0
stotal_download_bytes = 0
success_download_bytes = 0
total_sync_in_bytes = 0
success_sync_in_bytes = 0
total_sync_out_bytes = 0
success_sync_out_bytes = 0
total_file_open_count = 0
success_file_open_count = 0
total_file_read_count = 0
success_file_read_count = 0
total_file_write_count = 0
success_file_write_count = 0
last_heart_beat_time = 2015-05-27 20:19:44
last_source_update = 1969-12-31 16:00:00
last_sync_update = 1969-12-31 16:00:00
last_synced_timestamp = 1969-12-31 16:00:00
Storage 2:
id = 192.168.100.90
ip_addr = 192.168.100.90 (localhost)ACTIVE
http domain =
version = 5.06
join time = 2015-05-27 19:50:27
up time = 2015-05-27 19:50:27
total storage = 9916 MB
free storage = 7434 MB
upload priority = 10
store_path_count = 1
subdir_count_per_path = 256
storage_port = 23000
storage_http_port = 8080
current_write_path = 0
source storage id =
if_trunk_server = 0
connection.alloc_count = 256
connection.current_count = 1
connection.max_count = 1
total_upload_count = 0
success_upload_count = 0
total_append_count = 0
success_append_count = 0
total_modify_count = 0
success_modify_count = 0
total_truncate_count = 0
success_truncate_count = 0
total_set_meta_count = 0
success_set_meta_count = 0
total_delete_count = 0
success_delete_count = 0
total_download_count = 0
success_download_count = 0
total_get_meta_count = 0
success_get_meta_count = 0
total_create_link_count = 0
success_create_link_count = 0
total_delete_link_count = 0
success_delete_link_count = 0
total_upload_bytes = 0
success_upload_bytes = 0
total_append_bytes = 0
success_append_bytes = 0
total_modify_bytes = 0
success_modify_bytes = 0
stotal_download_bytes = 0
success_download_bytes = 0
total_sync_in_bytes = 0
success_sync_in_bytes = 0
total_sync_out_bytes = 0
success_sync_out_bytes = 0
total_file_open_count = 0
success_file_open_count = 0
total_file_read_count = 0
success_file_read_count = 0
total_file_write_count = 0
success_file_write_count = 0
last_heart_beat_time = 2015-05-27 20:19:48
last_source_update = 1969-12-31 16:00:00
last_sync_update = 1969-12-31 16:00:00
last_synced_timestamp = 1969-12-31 16:00:00
页:
[1]