44 分布式存储与mogilefs、mogiles使用进阶
01分布式存储与mogilefs实战：安装配置mogilefs
配置环境：
node1：192.168.1.130CentOS7.2
node2：192.168.1.131CentOS7.2
node3：192.168.1.132CentOS7.2
node4：192.168.1.133CentOS7.2
# vim /etc/hosts
添加
192.168.1.130 node1
192.168.1.131 node2
192.168.1.132 node3
192.168.1.133 node4
# scp /etc/hosts node2:/etc
# scp /etc/hosts node3:/etc
# scp /etc/hosts node4:/etc
# rpm -ivh epel-release-latest-7.noarch.rpm
# rpm -ivh epel-release-latest-7.noarch.rpm
# rpm -ivh epel-release-latest-7.noarch.rpm
# ls *rpm
MogileFS-Server-2.46-2.el6.noarch.rpm
MogileFS-Server-mogilefsd-2.46-2.el6.noarch.rpm
MogileFS-Server-mogstored-2.46-2.el6.noarch.rpm
MogileFS-Utils-2.19-1.el6.noarch.rpm
perl-Danga-Socket-1.61-1.el6.rf.noarch.rpm
perl-MogileFS-Client-1.14-1.el6.noarch.rpm
perl-Perlbal-1.78-1.el6.noarch.rpm
# yum install perl-Sys-Syslog perl-Net-Netmask perl-IO-AIO -y
# yum install *rpm -y
# scp *rpm node3:/root
# scp *rpm node4:/root
# yum install perl-Sys-Syslog perl-Net-Netmask perl-IO-AIO -y
# yum install *rpm -y
# yum install perl-Sys-Syslog perl-Net-Netmask perl-IO-AIO -y
# yum install *rpm -y
# yum -y install mariadb-server
# vim /etc/my.cnf
添加：
innodb_file_per_table = 1
skip_name_resolve = 1
# systemctl start mariadb.service
# systemctl enable mariadb.service
# mysql
MariaDB [(none)]> GRANT ALL ON *.* TO 'root'@'192.168.1.%' IDENTIFIED BY 'mageedu';
MariaDB [(none)]> CREATE DATABASE mogilefs;
MariaDB [(none)]> GRANT ALL ON mogilefs.* TO 'moguser'@'192.168.1.%' IDENTIFIED BY 'mogpass';
MariaDB [(none)]> FLUSH PRIVILEGES;
MariaDB [(none)]> \q
# mogdbsetup --dbhost=192.168.1.131 --dbrootpass='mageedu' --dbuser='moguser' --dbpass='mogpass'
# mysql
MariaDB [(none)]> use mogilefs;
MariaDB > SHOW TABLES;
+----------------------+
| Tables_in_mogilefs |
+----------------------+
| checksum |
| class |
| device |
| domain |
| file |
| file_on |
| file_on_corrupt |
| file_to_delete |
| file_to_delete2 |
| file_to_delete_later |
| file_to_queue |
| file_to_replicate |
| fsck_log |
| host |
| server_settings |
| tempfile |
| unreachable_fids |
+----------------------+
# cd /etc/mogilefs/
#Tracker节点配置
# vim mogilefsd.conf
修改
db_dsn = DBI:mysql:mogilefs:host=127.0.0.1
db_user = username
db_pass = password
为
db_dsn = DBI:mysql:mogilefs:host=192.168.1.131
db_user = moguser
db_pass = mogpass
修改
listen = 127.0.0.1:7001
为
listen = 0.0.0.0:7001
# service mogilefsdstart
#storage节点（可以是所有节点）
# mkdir -p /data/mogilefs/dev1
# chown -R mogilefs.mogilefs /data/mogilefs/
# vim /etc/mogilefs/mogstored.conf
修改
docroot = /var/mogdata
为
docroot = /data/mogilefs/
# vim /etc/init.d/mogstored
在lockfile=（15行左右）添加
pidfile=/var/run/mogilefsd/mogstored.pid
修改
[ $RETVAL = 0 ] && success && touch ${lockfile} || failure#约28行
为
[ $RETVAL = 0 ] && success && touch ${lockfile} && echo $(pidof mogstored) > ${pidfile} || failure
修改
[ $RETVAL = 0 ] && success && rm -f ${lockfile} || failure#约37行
为
[ $RETVAL = 0 ] && success && rm -f ${lockfile} ${pidfile} || failure
# service mogstored start
# mkdir -p /data/mogilefs/dev2
# chown -R mogilefs.mogilefs /data/mogilefs/
# mkdir -p /data/mogilefs/dev3
# chown -R mogilefs.mogilefs /data/mogilefs/
# scp /etc/rc.d/init.d/mogstored node3:/etc/rc.d/init.d/
# scp /etc/rc.d/init.d/mogstored node4:/etc/rc.d/init.d/
# scp /etc/mogilefs/mogstored.conf node3:/etc/mogilefs/
# scp /etc/mogilefs/mogstored.conf node4:/etc/mogilefs/
# systemctl start mogstored.service
# systemctl daemon-reload
# service mogstored restart
# systemctl daemon-reload
# service mogstored start
02mogiles使用进阶
mogilefs基本用法
#检测节点
# mogadm check
Checking trackers...
127.0.0.1:7001 ... OK
Checking hosts...
No devices found on tracker(s).
# mogadm --trackers=192.168.1.131:7001 check
Checking trackers...
192.168.1.131:7001 ... OK
Checking hosts...
No devices found on tracker(s).
#添加节点
# mogadm --trackers=192.168.1.131:7001 host add 192.168.1.131 --ip=192.168.1.131 --status=alive
# mogadm host list
192.168.1.131 : alive
IP: 192.168.1.131:7500
# mogadm --trackers=192.168.1.131:7001 host add 192.168.1.132 --ip=192.168.1.132 --status=alive
# mogadm --trackers=192.168.1.131:7001 host add 192.168.1.133 --ip=192.168.1.133 --status=alive
# mogadm host list
192.168.1.131 : alive
IP: 192.168.1.131:7500
192.168.1.132 : alive
IP: 192.168.1.132:7500
192.168.1.133 : alive
IP: 192.168.1.133:7500
#更改节点状态
# mogadm host mark 192.168.1.132 down
# mogadm host list
192.168.1.131 : alive
IP: 192.168.1.131:7500
192.168.1.132 : down
IP: 192.168.1.132:7500
192.168.1.133 : alive
IP: 192.168.1.133:7500
# mogadm host mark 192.168.1.132 alive
# mogadm host list
192.168.1.131 : alive
IP: 192.168.1.131:7500
192.168.1.132 : alive
IP: 192.168.1.132:7500
192.168.1.133 : alive
IP: 192.168.1.133:7500
#查看设备
# mogadm device list
192.168.1.131 : alive
used(G) free(G) total(G)weight(%)
192.168.1.132 : alive
used(G) free(G) total(G)weight(%)
192.168.1.133 : alive
used(G) free(G) total(G)weight(%)
#添加设备
# mogadm device add 192.168.1.131 1
# mogadm device list
192.168.1.131 : alive
used(G) free(G) total(G)weight(%)
dev1: alive 4.375 15.615 19.990 100
192.168.1.132 : alive
used(G) free(G) total(G)weight(%)
192.168.1.133 : alive
used(G) free(G) total(G)weight(%)
# mogadm device add 192.168.1.132 2
# mogadm device add 192.168.1.133 3
# mogadm device list
192.168.1.131 : alive
used(G) free(G) total(G)weight(%)
dev1: alive 4.375 15.615 19.990 100
192.168.1.132 : alive
used(G) free(G) total(G)weight(%)
dev2: alive 4.246 15.744 19.990 100
192.168.1.133 : alive
used(G) free(G) total(G)weight(%)
dev3: alive 4.246 15.744 19.990 100
# mogadm check
Checking trackers...
127.0.0.1:7001 ... OK
Checking hosts...
[ 1] 192.168.1.131 ... OK
[ 2] 192.168.1.132 ... OK
[ 3] 192.168.1.133 ... OK
Checking devices...
host device size(G) used(G) free(G) use% ob state I/O%
---- ------------ ---------- ---------- ---------- ------ ---------- -----
[ 1] dev1 19.990 4.376 15.61421.89%writeable 0.0
[ 2] dev2 19.990 4.246 15.74421.24%writeable 0.0
[ 3] dev3 19.990 4.246 15.74421.24%writeable 0.0
---- ------------ ---------- ---------- ---------- ------
total: 59.971 12.869 47.10221.46%
# mogadm domain add images
# mogadm domain add files
# mogadm domain list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
# mogadm class add images jpeg
# mogadm class add images png
# mogadm class add images gif
# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
images gif 2 MultipleHosts() NONE
images jpeg 2 MultipleHosts() NONE
images png 2 MultipleHosts() NONE
# mogadm class add files plaintext --mindevcount=1
# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
files plaintext 1 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
images gif 2 MultipleHosts() NONE
images jpeg 2 MultipleHosts() NONE
images png 2 MultipleHosts() NONE
# mogadm class add files html --replpolicy="MultipleHosts(3)"
# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
files html 2 MultipleHosts(3) NONE
files plaintext 1 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
images gif 2 MultipleHosts() NONE
images jpeg 2 MultipleHosts() NONE
images png 2 MultipleHosts() NONE
#上传文件
# mogupload --trackers=192.168.1.131:7001 --domain=files --key='/fstab.txt' --file='/etc/fstab' --class=plaintext
# mogfileinfo --trackers=192.168.1.131:7001 --domain=files --key='/fstab.txt'- file: /fstab.txt
class: plaintext
devcount: 1
domain: files
fid: 2
key: /fstab.txt
length: 465
- http://192.168.1.131:7500/dev1/0/000/000/0000000002.fid
#可以通过上面网址访问文件
# mogupload --trackers=192.168.1.131:7001 --domain=images --class=jpeg --key='/linux.jpg' --file='./01.jpg'
# mogfileinfo --trackers=192.168.1.131:7001 --domain=images --key='/linux.jpg'- file: /linux.jpg
class: jpeg
devcount: 1
domain: images
fid: 3
key: /linux.jpg
length: 60582
- http://192.168.1.132:7500/dev2/0/000/000/0000000003.fid
# mogupload --trackers=192.168.1.131:7001 --domain=images--key='/linux2.jpg' --file='./02.jpg'
# mogfileinfo --trackers=192.168.1.131:7001 --domain=images --key='/linux2.jpg'
- file: /linux2.jpg
class: default
devcount: 1
domain: images
fid: 4
key: /linux2.jpg
length: 116463
- http://192.168.1.132:7500/dev2/0/000/000/0000000004.fid
#查看上传的文件
# moglistkeys --trackers=192.168.1.131:7001 --domain=images
/linux.jpg
/linux2.jpg
# moglistkeys --trackers=192.168.1.131:7001 --domain=files
/fstab.txt
# moglistfids --trackers=192.168.1.131:7001 --domain=files
fid 2
class plaintext
devcount 1
domain files
key /fstab.txt
length 465
fid 3
class jpeg
devcount 1
domain images
key /linux.jpg
length 60582
fid 4
class default
devcount 1
domain images
key /linux2.jpg
length 116463
# mogstats --db_dsn="DBI:mysql:mogilefs:host=192.168.1.131" --db_user="moguser" --db_pass="mogpass" --verbose --stats="devices,files"
Fetching statistics... (devices,files)
... files stats...
... done
... per-device stats...
... done
Statistics for devices...
device host files status
---------- ---------------- ------------ ----------
dev1 192.168.1.131 1 alive
dev2 192.168.1.132 2 alive
---------- ---------------- ------------ ----------
Statistics for files...
domain class files size (m)fullsize (m)
-------------------- ----------- ---------- ----------- -------------
files plaintext 1 0 0
images default 1 0 0
images jpeg 1 0 0
-------------------- ----------- ---------- ----------- -------------
done
# mogstats --db_dsn="DBI:mysql:mogilefs:host=192.168.1.131" --db_user="moguser" --db_pass="mogpass" --verbose --stats="all"
Fetching statistics... (all)
... replication stats...
... done
... replication queue stats...
... done
... delete queue stats...
... done
... files stats...
... done
... per-device stats...
... done
... fid stats...
... done
... queue stats...
... done
Statistics for devices...
device host files status
---------- ---------------- ------------ ----------
dev1 192.168.1.131 1 alive
dev2 192.168.1.132 2 alive
---------- ---------------- ------------ ----------
Statistics for file ids...
Max file id: 4
Statistics for files...
domain class files size (m)fullsize (m)
-------------------- ----------- ---------- ----------- -------------
files plaintext 1 0 0
images default 1 0 0
images jpeg 1 0 0
-------------------- ----------- ---------- ----------- -------------
Statistics for replication...
domain class devcount files
-------------------- ----------- ---------- ----------
files plaintext 1 1
images default 1 1
images jpeg 1 1
-------------------- ----------- ---------- ----------
Statistics for replication queue...
status count
-------------------- ------------
deferred 2
-------------------- ------------
Statistics for delete queue...
status count
-------------------- ------------
-------------------- ------------
Statistics for general queues...
queue status count
--------------- -------------------- ------------
--------------- -------------------- ------------
done
#删除文件
# mogdelete --trackers=192.168.1.131:7001 --domain=images--key='/linux.jpg'
# mogdelete --trackers=192.168.1.131:7001 --domain=images--key='/linux2.jpg'
# mogdelete --trackers=192.168.1.131:7001 --domain=files--key='/fstab.txt'
#
# mogadm class delete images jpeg
# mogadm class delete images png
# mogadm class delete images gif
# mogadm class delete files plaintext
# mogadm class delete files html
# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
# mogadm domain delete files
# mogadm domain delete images
# mogadm domain add imgs
# mogadm domain list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
imgs default 2 MultipleHosts() NONE
# mogupload --trackers=192.168.1.131:7001 --domain=imgs --key='linux.jpg' --file='./01.jpg'
# mogfileinfo --trackers=192.168.1.131:7001 --domain=imgs --key='linux.jpg'
- file: linux.jpg
class: default
devcount: 1
domain: imgs
fid: 5
key: linux.jpg
length: 60582
- http://192.168.1.131:7500/dev1/0/000/000/0000000005.fid
# mogadm class add imgs jpeg --mindevcount=2 --replpolicy="MultipleHosts(2)"# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
imgs default 2 MultipleHosts() NONE
imgs jpeg 2 MultipleHosts(2) NONE
# mogupload --trackers=192.168.1.131:7001 --domain=imgs --class=jpeg --key='linux2.jpg' --file='./02.jpg'
# mogfileinfo --trackers=192.168.1.131:7001 --domain=imgs --key='linux2.jpg'- file: linux2.jpg
class: jpeg
devcount: 1
domain: imgs
fid: 6
key: linux2.jpg
length: 116463
- http://192.168.1.131:7500/dev1/0/000/000/0000000006.fid
实战：Nginx做为MogileFS的前端客户端
一、安装Nginx：
1、解决依赖关系
# yum groupinstall "Development Tools" "Server Platform Deveopment"
# yum -y install openssl-devel pcre-devel
2、安装
首先添加用户nginx，实现以之运行nginx服务进程：
# groupadd -r nginx
# useradd -r -g nginx nginx
# tar xf nginx_mogilefs_module-1.0.4.tar.gz
# tar xf tengine-2.0.1.tar.gz
# cd tengine-2.0.1/
# ./configure \
--prefix=/usr \
--sbin-path=/usr/sbin/nginx \
--conf-path=/etc/nginx/nginx.conf \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--pid-path=/var/run/nginx/nginx.pid\
--lock-path=/var/lock/nginx.lock \
--user=nginx \
--group=nginx \
--with-http_ssl_module \
--with-http_flv_module \
--with-http_stub_status_module \
--with-http_gzip_static_module \
--http-client-body-temp-path=/var/tmp/nginx/client/ \
--http-proxy-temp-path=/var/tmp/nginx/proxy/ \
--http-fastcgi-temp-path=/var/tmp/nginx/fcgi/ \
--http-uwsgi-temp-path=/var/tmp/nginx/uwsgi \
--http-scgi-temp-path=/var/tmp/nginx/scgi \
--with-pcre \
--with-debug \
--add-module=../nginx_mogilefs_module-1.0.4
# vim objs/Makefile
修改
CFLAGS =-pipe-O -W -Wall -Wpointer-arith -Wno-unused-parameter -Werror -g
为
CFLAGS =-pipe-O -W -Wall -Wpointer-arith -Wno-unused-parameter -g
# make && make install
# vim /usr/lib/systemd/system/nginx.service
Description=The nginx HTTP and reverse proxy server
After=network.target remote-fs.target nss-lookup.target
Type=forking
PIDFile=/var/run/nginx/nginx.pid
ExecStartPre=/usr/sbin/nginx -t
ExecStart=/usr/sbin/nginx
ExecReload=/bin/kill -s HUP $MAINPID
KillMode=process
KillSignal=SIGQUIT
TimeoutStopSec=5
PrivateTmp=true
WantedBy=multi-user.target
# systemctl daemon-reload
# systemctl start nginx.service
#测试不成功
# cd /etc/nginx/
# cp nginx.conf{,.bak}
# vim nginx.conf
添加
location/images/ {
mogilefs_tracker 192.168.1.131:7001;
mogilefs_domain imgs;
mogilefs_methods GET;
mogilefs_noverify on;
mogilefs_pass {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
# nginx -t
# systemctl restart nginx.service
# moglistkeys --trackers=192.168.1.131:7001 --domain=imgs
linux.jpg
linux2.jpg
# mogdelete --trackers=192.168.1.131:7001 --domain=imgs --key='linux.jpg'
# mogdelete --trackers=192.168.1.131:7001 --domain=imgs --key='linux2.jpg'
# mogupload --trackers=192.168.1.131:7001 --domain=imgs --key='linux.jpg' --file='./01.jpg'
# systemctl restart nginx.service
# mogupload --trackers=192.168.1.131:7001 --domain=imgs --key='linux2.jpg' --file='./02.jpg'
# scp /etc/mogilefs/mogilefsd.conf node3:/etc/mogilefs/
# scp /etc/mogilefs/mogilefsd.conf node4:/etc/mogilefs/
# systemctl start mogilefsd.service
# moglistkeys --trackers=192.168.1.132:7001 --domain=imgs
linux.jpg
linux2.jpg
# systemctl start mogilefsd.service
# moglistkeys --trackers=192.168.1.133:7001 --domain=imgs
linux.jpg
linux2.jpg
# vim /etc/nginx/nginx.conf
在38行左右添加
upstream trackers {
server 192.168.1.131:7001;
server 192.168.1.132:7001;
server 192.168.1.133:7001;
check interval=1000 rise=2 fall=5 timeout=1000;
}
修改location为
location/images/ {
mogilefs_tracker trackers;
mogilefs_domain imgs;
mogilefs_methods GET;
mogilefs_noverify on;
mogilefs_pass {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
location /status {
check_status;
}
# systemctl restart nginx.service
64:43(58848)
页:
[1]