Heartbeat+DRBD+MFS高可用
一、环境系统 CentOS 6.4x64最小化安装
mfs-master 192.168.3.33
mfs-slave 192.168.3.34
vip 192.168.3.35
mfs-chun 192.168.3.36
mfs-client 192.168.3.37
二、基础配置
#关闭iptables，配置hosts本地解析
# service iptables stop
# vim /etc/hosts
# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.3.33mfs-master
192.168.3.34mfs-slave
192.168.3.36mfs-chun
192.168.3.37mfs-client
#安装yum源
# rpm -ivh http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
Retrieving http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
warning: /var/tmp/rpm-tmp.sh3hsI: Header V3 RSA/SHA256 Signature, key ID 0608b895: NOKEY
Preparing... ###########################################
1:epel-release ###########################################
# sed -i 's@#b@b@g' /etc/yum.repos.d/epel.repo
# sed-i 's@mirrorlist@#mirrorlist@g' /etc/yum.repos.d/epel.repo
# rpm -ivh http://elrepo.org/elrepo-release-6-5.el6.elrepo.noarch.rpm
#配置ntp同步时间
# echo "*/10 * * * * /usr/sbin/ntpdate asia.pool.ntp.org&>/dev/null" >/var/spool/cron/root
#配置ssh互信(这里只需要mfs-master和mfs-slave互信就行了)
#mfs-master操作
# ssh-keygen
# ssh-copy-id -i ~/.ssh/id_rsa.pub root@mfs-slave
#mfs-slave操作
# ssh-keygen
# ssh-copy-id -i ~/.ssh/id_rsa.pub root@mfs-master 三、安装配置heartbeat
(1).在mfs-master和mfs-slave执行同样的安装操作
# yum install heartbeat -y (2).配置ha.cf
# egrep -v "^$|^#" /etc/ha.d/ha.cf
logfile/var/log/ha-log
logfacilitylocal1
keepalive 2
deadtime 30
warntime 10
initdead 120
mcast eth0 225.0.10.1 694 1 0
auto_failback on
node mfs-master
node mfs-slave
crm no (3).配置authkeys
# dd if=/dev/random bs=512 count=1 |openssl md5
0+1 records in
0+1 records out
21 bytes (21 B) copied, 5.0391e-05 s, 417 kB/s
(stdin)= c55529482f1c76dd8967ba41f5441ae1
# grep -v ^# /etc/ha.d/authkeys
auth 1
1 md5 c55529482f1c76dd8967ba41f5441ae1
# chmod 600 /etc/ha.d/authkeys (4).配置haresource
# grep -v ^# /etc/ha.d/haresources
mfs-masterIPaddr::192.168.3.35/24/eth0 #暂时只配置一个IP资源用于调试 (5).启动heartbeat
#说明，工作中应关闭开机自启动，当服务器重启时，由人工手动启动
# chkconfig heartbeat off
#启动服务
# /etc/init.d/heartbeat start
Starting High-Availability services: INFO:Resource is stopped
Done.
#将配置文件复制到mfs-slave上
# scp authkeys ha.cf haresources mfs-slave:/etc/ha.d
#在mfs-slave上启动服务
# /etc/init.d/heartbeat start
Starting High-Availability services: INFO:Resource is stopped
Done.
#查看结果
# ip a|grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.33/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.35/24 brd 192.168.3.255 scope global secondary eth0
#查询备节点上的ip信息，备几点没有ip
# ssh mfs-slave ip a|grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.34/24 brd 192.168.3.255 scope global eth0
(6).测试heartbeat
正常状态
#mfs-master的IP信息
# ip a |grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.33/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.35/24 brd 192.168.3.255 scope global secondary eth0
#mfs-slave的IP信息
# ip a |grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.34/24 brd 192.168.3.255 scope global eth0 模拟主节点宕机后的状态信息
#在主节点mfs-master上停止heartbeat服务
# /etc/init.d/heartbeat stop
Stopping High-Availability services: Done.
# ip a|grep eth0 主节点的heartbeat服务停止后，vip资源被抢走
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.33/24 brd 192.168.3.255 scope global eth0
#在备节点mfs--slave查看资源
# ip a |grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.34/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.35/24 brd 192.168.3.255 scope global secondary eth0 恢复主节点的heartbeat服务
# /etc/init.d/heartbeat start
Starting High-Availability services: INFO:Resource is stopped
Done.
#主节点的heartbeat服务恢复后，将资源接管回来了
# ip a |grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.33/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.35/24 brd 192.168.3.255 scope global secondary eth0
# ip a |grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.34/24 brd 192.168.3.255 scope global eth0 四、安装部署DRBD
(1).对硬盘进行分区，mfs-master和mfs-slave的操作一样
# fdisk /dev/sdb
#说明：/dev/sdb分成2个分区/dev/sdb1和/dev/sdb2，/dev/sdb1=15G
# partprobe /dev/sdb
#对分区进行格式化
# mkfs.ext4 /dev/sdb1
说明：sdb2分区为meta data分区，不需要格式化操作
# tune2fs -c -1 /dev/sdb1
说明：设置最大挂载数为-1,关闭强制检查挂载次数限制 2).安装DRBD
由于我们的系统是CentOS6.4的，所以我们还需要安装内核模块，版本需要和uname -r保持一致，安装包我们从系统安装软件中提取出来，过程略。mfs-master和mfs-slave的安装过程一样，这里只给出mfs-master的安装过程
#安装系统内核文件
# rpm -ivh kernel-devel-2.6.32-358.el6.x86_64.rpm kernel-headers-2.6.32-358.el6.x86_64.rpm
# yum install drbd84 kmod-drbd84 -y
(3).配置DRBD
a.修改全局配置文件
# egrep -v "^$|^#|^[[:space:]]+#" /etc/drbd.d/global_common.conf
global {
usage-count no;
}
common {
protocol C;
handlers {
}
startup {
}
options {
}
disk {
on-io-error detach;
no-disk-flushes;
no-md-flushes;
rate 200M;
}
net {
sndbuf-size 512k;
max-buffers 8000;
unplug-watermark 1024;
max-epoch-size8000;
cram-hmac-alg "sha1";
shared-secret "weyee2014";
after-sb-0pri disconnect;
after-sb-1pri disconnect;
after-sb-2pri disconnect;
rr-conflict disconnect;
}
} b.增加资源
# cat /etc/drbd.d/mfsdata.res
resource mfsdata {
on mfs-master {
device /dev/drbd1;
disk /dev/sdb1;
address 192.168.3.33:7789;
meta-disk /dev/sdb2 ;
}
on mfs-slave {
device /dev/drbd1;
disk/dev/sdb1;
address 192.168.3.34:7789;
meta-disk /dev/sdb2 ;
}
} c.将配置文件复制到ha-node2上，重启系统加载drbd模块，初始化meta数据
# scp global_common.conf mfsdata.res mfs-slave:/etc/drbd.d/
# depmod
# modprobe drbd #我这是使用虚拟机安装的，重启后才能加载到drbd模块，不知道这是为什么
# lsmod |grep drbd
drbd 3659312
libcrc32c 12461 drbd
#在mfs-master上初始化meta数据
# drbdadm create-md mfsdata
initializing activity log
NOT initializing bitmap
Writing meta data...
New drbd meta data block successfully created.
#在mfs-slave上加载模块，初始化meta数据
# depmod
# modprobe drbd
# lsmod |grep drbd
drbd 3659310
libcrc32c 12461 drbd
# drbdadm create-md mfsdata
initializing activity log
NOT initializing bitmap
Writing meta data...
New drbd meta data block successfully created. d.在mfs-master和mfs-slave上启动drbd
#mfs-master操作
# /etc/init.d/drbd start
#mfs-slave操作
# /etc/init.d/drbd start
# drbd-overview
1:mfsdata/0Connected Secondary/Secondary Inconsistent/Inconsistent
#将mfs-master设置成主节点
# drbdadm -- --overwrite-data-of-peer primary mfsdata
# drbd-overview
1:mfsdata/0SyncSource Primary/Secondary UpToDate/Inconsistent
[>....................] sync'ed:1.4% (15160/15364)M
#将DRBD设备挂载到/data目录下，写入测试数据mfs-master.txt
# mount /dev/drbd1 /data
# touch /data/mfs-master.txt
# ls /data/
lost+foundmfs-master.txt
#状态结果显示UpToDate/UpToDate表示主备节点数据已同步
# drbd-overview
1:mfsdata/0Connected Primary/Secondary UpToDate/UpToDate /data ext4 15G 38M 14G 1% e.测试DRBD
正常状态
# drbd-overview
1:mfsdata/0Connected Primary/Secondary UpToDate/UpToDate /data ext4 15G 38M 14G 1%
#注：这里显示的是mfs-master是主节点，mfs-slave是从节点 模拟宕机后的状态
# umount /data
#将mfs-master设置成secondary状态
# drbdadm secondary mfsdata
# drbd-overview
1:mfsdata/0Connected Secondary/Secondary UpToDate/UpToDate
#将mfs-slave设置成primary状态
# drbdadm primary mfsdata
# drbd-overview
1:mfsdata/0Connected Primary/Secondary UpToDate/UpToDate
# mount /dev/drbd1 /mnt
#查看文件，测试结果正常
# ls /mnt
lost+foundmfs-master.txt
#注：DRBD主节点宕机后，将备节点设置成primary状态后能正常使用，且数据一致
#将DRBD状态恢复成原状态 五、在mfs-master上安装MFS
因为我们要使用drbd的高可用，所有我们的drbd设备要挂载到/usr/local/mfs目录下
# mkdir /usr/local/mfs
# mount /dev/drbd1 /usr/local/mfs/
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.4G 16G 9% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 190M 48M132M27% /boot
/dev/drbd1 15G 38M 14G 1% /usr/local/mfs MFS的安装部分mfs-master
#开始安装MFS
# yum install zlib-devel -y
# groupadd -g 1000 mfs
# useradd -u 1000 -g mfs -s /sbin/nologin mfs
# wget http://moosefs.org/tl_files/mfscode/mfs-1.6.27-5.tar.gz
# tar xf mfs-1.6.27-5.tar.gz
# cd mfs-1.6.27
# ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs--disable-mfschunkserver --disable-mfsmount
# make && make install
#配置mfs-master配置文件
# cp mfsexports.cfg.dist mfsexports.cfg
# cp mfsmaster.cfg.dist mfsmaster.cfg
# egrep -v "^#|^$" /usr/local/mfs/etc/mfs/mfsexports.cfg
*/rw,alldirs,mapall=mfs:mfs,password=centos
*.rw
# cp /usr/local/mfs/var/mfs/metadata.mfs.empty /usr/local/mfs/var/mfs/metadata.mfs
#启动mfs
# /usr/local/mfs/sbin/mfsmaster start
working directory: /usr/local/mfs/var/mfs
lockfile created and locked
initializing mfsmaster modules ...
loading sessions ... file not found
if it is not fresh installation then you have to restart all active mounts !!!
exports file has been loaded
mfstopology configuration file (/usr/local/mfs/etc/mfstopology.cfg) not found - using defaults
loading metadata ...
create new empty filesystemmetadata file has been loaded
no charts data file - initializing empty charts
mastermetaloggers module: listen on *:9419
masterchunkservers module: listen on *:9420
main master server module: listen on *:9421
mfsmaster daemon initialized properly
#查看结果
# ps aux |grep mfs |grep -v grep
root 13300.00.0 0 0 ? S 09:53 0:00
root 13440.00.0 0 0 ? S 09:53 0:00
root 13490.00.0 0 0 ? S 09:53 0:00
mfs 69900.3 18.8 108628 92996 ? S< 10:15 0:00 /usr/local/mfs/sbin/mfsmaster start
# netstat -tunlp |grep mfs
tcp 0 0 0.0.0.0:9419 0.0.0.0:* LISTEN 6990/mfsmaster
tcp 0 0 0.0.0.0:9420 0.0.0.0:* LISTEN 6990/mfsmaster
tcp 0 0 0.0.0.0:9421 0.0.0.0:* LISTEN 6990/mfsmaster
#停止mfs
# /usr/local/mfs/sbin/mfsmaster stop
sending SIGTERM to lock owner (pid:6990)
waiting for termination ... terminated
#最后配置环境变量
# echo '# add moosefs to the path variable' >> /etc/profile
# echo 'PATH=/usr/local/mfs/sbin/:$PATH' >> /etc/profile
# tail -2 /etc/profile
# add moosefs to the path variable
PATH=/usr/local/mfs/sbin/:$PATH
# source /etc/profile
#为heartbeat启动mfs复制文件
# cp /usr/local/mfs/sbin/mfsmaster /etc/init.d/
#最后操作，卸载文件系统
# drbd-overview
1:mfsdata/0Connected Primary/Secondary UpToDate/UpToDate /usr/local/mfs ext4 15G 41M 14G 1%
# umount /usr/local/mfs/
#由于我们是要通过heartbeat来启动mfs，所以DRBD应该交给heartbeat进行挂载 六、mfs-slave配置
因为mfs-slave是使用的drbd,所以我们只需要中一些简单的操作就可以启动mfs了
#让DRBD设备下线
# drbdadm down mfsdata
#将/dev/sdb1分区挂载到/usr/local/mfs目录下
# mkdir -p /usr/local/mfs
# mount /dev/sdb1 /usr/local/mfs
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.6G 16G10% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 190M 48M132M27% /boot
/dev/sdb1 15G 41M 14G 1% /usr/local/mfs
#安装基础软件
# yum install zlib-devel -y
#创建用户和组
# groupadd -g 1000 mfs
# useradd -u 1000 -g mfs -s /sbin/nologin mfs
#复制启动文件到/etc/init.d
# cp /usr/local/mfs/sbin/mfsmaster /etc/init.d/
#测试启动脚本
# /etc/init.d/mfsmaster start
working directory: /usr/local/mfs/var/mfs
lockfile created and locked
initializing mfsmaster modules ...
loading sessions ... ok
sessions file has been loaded
exports file has been loaded
mfstopology configuration file (/usr/local/mfs/etc/mfstopology.cfg) not found - using defaults
loading metadata ...
loading objects (files,directories,etc.) ... ok
loading names ... ok
loading deletion timestamps ... ok
loading chunks data ... ok
checking filesystem consistency ... ok
connecting files and chunks ... ok
all inodes: 1
directory inodes: 1
file inodes: 0
chunks: 0
metadata file has been loaded
stats file has been loaded
mastermetaloggers module: listen on *:9419
masterchunkservers module: listen on *:9420
main master server module: listen on *:9421
mfsmaster daemon initialized properly
# ps aux |grep mfsmaster
mfs 24090.5 18.8 108628 93008 ? S< 10:28 0:00 /etc/init.d/mfsmaster start
root 24110.00.1 103248 876 pts/0 S+ 10:28 0:00 grep mfsmaster
# netsat -tunlp |grep mfs
-bash: netsat: command not found
# netstat -tunlp |grep mfs
tcp 0 0 0.0.0.0:9419 0.0.0.0:* LISTEN 2409/mfsmaster
tcp 0 0 0.0.0.0:9420 0.0.0.0:* LISTEN 2409/mfsmaster
tcp 0 0 0.0.0.0:9421 0.0.0.0:* LISTEN 2409/mfsmaster
#关闭服务
# /etc/init.d/mfsmaster stop
sending SIGTERM to lock owner (pid:2409)
waiting for termination ... terminated
#配置环境变量
# echo '# add moosefs to the path variable' >> /etc/profile
# echo 'PATH=/usr/local/mfs/sbin/:$PATH' >> /etc/profile
#tail -2 /etc/profile
# add moosefs to the path variable
PATH=/usr/local/mfs/sbin/:$PATH
# source /etc/profile
#最后将文件系统卸载，让DRBD上线
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.6G 16G10% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 190M 48M132M27% /boot
/dev/sdb1 15G 41M 14G 1% /usr/local/mfs
# umount /usr/local/mfs/
# drbdadm up mfsdata
# drbd-overview
1:mfsdata/0Connected Secondary/Primary UpToDate/UpToDate 七、配置heartbeat管理相关服务
#在前面我们已经配置了通过heartbeat来管理vip资源。现在我们配置管理DRBD资源
#修改配置文件/etc/ha.d/haresources
# egrep -v "^#|^$" /etc/ha.d/haresources
mfs-masterIPaddr::192.168.3.35/24/eth0 drbddisk::mfsdata Filesystem::/dev/drbd1::/usr/local/mfs::ext4
# scp /etc/ha.d/haresources mfs-slave:/etc/ha.d/haresources
#重启mfs-mastar和mfs-slave的heartbeat服务
# ip a|grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.33/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.35/24 brd 192.168.3.255 scope global secondary eth0
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.6G 16G10% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 190M 48M132M27% /boot
/dev/drbd1 15G 41M 14G 1% /usr/local/mfs
#查看mfs-slave的情况
# ip a |grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.34/24 brd 192.168.3.255 scope global eth0
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.6G 16G10% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 190M 48M132M27% /boot
#停止mfs-master的heartbeat服务，能正常接管服务
# ip a |grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.34/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.35/24 brd 192.168.3.255 scope global secondary eth0
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.6G 16G10% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 190M 48M132M27% /boot
/dev/drbd1 15G 41M 14G 1% /usr/local/mfs
#再将mfs-master的heartbeat服务打开
# /etc/init.d/heartbeat start
Starting High-Availability services: INFO:Resource is stopped
Done.
#能够正常接管会资源
# ip a|grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.33/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.35/24 brd 192.168.3.255 scope global secondary eth0
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.6G 16G10% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 190M 48M132M27% /boot
/dev/drbd1 15G 41M 14G 1% /usr/local/mfs
#最后将mfs服务交给heartbeat管理
# egrep -v "^$|^#" /etc/ha.d/haresources
mfs-masterIPaddr::192.168.3.35/24/eth0 drbddisk::mfsdata Filesystem::/dev/drbd1::/usr/local/mfs::ext4mfsmaster
# scp /etc/ha.d/haresources mfs-slave:/etc/ha.d/haresources
#重启mfs-master的heartbear服务的情况
# ip a |grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.33/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.35/24 brd 192.168.3.255 scope global secondary eth0
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.6G 16G10% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 190M 48M132M27% /boot
/dev/drbd1 15G 41M 14G 1% /usr/local/mfs
# netstat -tulp|grep mfs
tcp 0 0 *:9419 *:* LISTEN 12888/mfsmaster
tcp 0 0 *:9420 *:* LISTEN 12888/mfsmaster
tcp 0 0 *:9421 *:* LISTEN 12888/mfsmaster
#停止mfs-master的heartbeat服务
# /etc/init.d/heartbeat stop
Stopping High-Availability services: Done.
#查看mfs-slave的信息
# ip a|grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.34/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.35/24 brd 192.168.3.255 scope global secondary eth0
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.6G 16G10% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 190M 48M132M27% /boot
/dev/drbd1 15G 41M 14G 1% /usr/local/mfs
# netstat -tunlp|grep mfs
tcp 0 0 0.0.0.0:9419 0.0.0.0:* LISTEN 8664/mfsmaster
tcp 0 0 0.0.0.0:9420 0.0.0.0:* LISTEN 8664/mfsmaster
tcp 0 0 0.0.0.0:9421 0.0.0.0:* LISTEN 8664/mfsmaster
#再重新启动mfs-master的heartbeat服务
# /etc/init.d/heartbeat start
Starting High-Availability services: INFO:Resource is stopped
Done.
# ip a|grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.33/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.35/24 brd 192.168.3.255 scope global secondary eth0
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.6G 16G10% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 190M 48M132M27% /boot
/dev/drbd1 15G 41M 14G 1% /usr/local/mfs
# netstat -tunlp|grep mfs
tcp 0 0 0.0.0.0:9419 0.0.0.0:* LISTEN 13845/mfsmaster
tcp 0 0 0.0.0.0:9420 0.0.0.0:* LISTEN 13845/mfsmaster
tcp 0 0 0.0.0.0:9421 0.0.0.0:* LISTEN 13845/mfsmaster 八、安装mfs-chun(整个环境是在vmware中完成的，所以只用了一台chunkserver)
# yum install zlib-devel gcc gcc-c++ make -y
# groupadd -g 1000 mfs
# useradd -u 1000 -g mfs mfs -s /sbin/nologin
# tar xf mfs-1.6.27-5.tar.gz
# cd mfs-1.6.27
# ./configure --prefix=/usr/local/mfs-1.6.27 --with-default-user=mfs --with-default-group=mfs--disable-mfsmaster --disable-mfsmount
# make && make install
# ln -s /usr/local/mfs-1.6.27 /usr/local/mfs
# ll -d /usr/local/mfs
lrwxrwxrwx 1 root root 21 Jul 23 09:49 /usr/local/mfs -> /usr/local/mfs-1.6.27 编辑mfs-chun的配置文件
# cd /usr/local/mfs/etc/mfs/
# cp mfschunkserver.cfg.dist mfschunkserver.cfg
# cp mfshdd.cfg.dist mfshdd.cfg
# egrep -v "^#|^$" mfschunkserver.cfg
MASTER_HOST = 192.168.3.35 #该地址是前面架构中的vip地址
#配置mfshdd.cfg
#这里我们将/dev/sdb格式化，然后挂载到mfsdata下
# fdisk /dev/sdb
# mkfs.ext4 /dev/sdb
# mkdir /mfsdata
# chown -R mfs:mfs /mfsdata #mfs用户需要有写权限
# mount /dev/sdb /mfsdata
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.5G 16G 9% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 194M 28M156M16% /boot
/dev/sdb 99G188M 94G 1% /mfsdata
#修改mfshdd.cfg，将/mfsdata交给mfs-master管理
# vim /usr/local/mfs/etc/mfs/mfshdd.cfg
/mfsdata
#启动mfs-chun的服务
#测试mfs-master的服务是否起来
# nc -w 2 192.168.3.35 -z 9420
Connection to 192.168.3.35 9420 port succeeded!
# /usr/local/mfs/sbin/mfschunkserver start
working directory: /usr/local/mfs-1.6.27/var/mfs
lockfile created and locked
initializing mfschunkserver modules ...
hdd space manager: path to scan: /mfsdata/
hdd space manager: start background hdd scanning (searching for available chunks)
main server module: listen on *:9422
no charts data file - initializing empty charts
mfschunkserver daemon initialized properly
# netstat -lanpt |grep 9420
tcp 0 0 192.168.3.36:51814 192.168.3.35:9420 ESTABLISHED 5948/mfschunkserver
#查看mfs-master的日志
# tail /var/log/messages
Jul 23 09:45:41 mfs-master mfsmaster: open files limit: 5000
Jul 23 09:45:41 mfs-master heartbeat: : info: local HA resource acquisition completed (standby).
Jul 23 09:45:41 mfs-master heartbeat: : info: Standby resource acquisition done .
Jul 23 09:45:41 mfs-master heartbeat: : info: Initial resource acquisition complete (auto_failback)
Jul 23 09:45:41 mfs-master heartbeat: : info: remote resource transition completed.
Jul 23 09:58:30 mfs-master mfsmaster: connection with CS(192.168.3.36) has been closed by peer
Jul 23 09:58:30 mfs-master mfsmaster: chunkserver disconnected - ip: 192.168.3.36, port: 0, usedspace: 0 (0.00 GiB), totalspace: 0 (0.00 GiB)
Jul 23 09:59:48 mfs-master mfsmaster: chunkserver register begin (packet version: 5) - ip: 192.168.3.36, port: 9422
Jul 23 09:59:48 mfs-master mfsmaster: chunkserver register end (packet version: 5) - ip: 192.168.3.36, port: 9422, usedspace: 0 (0.00 GiB), totalspace: 0 (0.00 GiB)
Jul 23 10:00:00 mfs-master mfsmaster: no meta loggers connected !!!
#测试：将mfs-master的heartbeat服务停止
# /etc/init.d/heartbeat stop
Stopping High-Availability services: Done.
#在mfs-slave上查看连接信息
# netstat -anpt |grep mfs
tcp 0 0 0.0.0.0:9419 0.0.0.0:* LISTEN 3195/mfsmaster
tcp 0 0 0.0.0.0:9420 0.0.0.0:* LISTEN 3195/mfsmaster
tcp 0 0 0.0.0.0:9421 0.0.0.0:* LISTEN 3195/mfsmaster
tcp 0 0 192.168.3.35:9420 192.168.3.36:51815 ESTABLISHED 3195/mfsmaster
#可以看到mfs-chun一直处于连接状态
#再将mfs-master的heartbeat服务重启启动
# netstat -anpt |grep mfs
tcp 0 0 0.0.0.0:9419 0.0.0.0:* LISTEN 3123/mfsmaster
tcp 0 0 0.0.0.0:9420 0.0.0.0:* LISTEN 3123/mfsmaster
tcp 0 0 0.0.0.0:9421 0.0.0.0:* LISTEN 3123/mfsmaster
tcp 0 0 192.168.3.35:9420 192.168.3.36:51817 ESTABLISHED 3123/mfsmaster
#最后配置环境变量
# echo '# add moosefs to the path variable' >> /etc/profile
# echo 'PATH=/usr/local/mfs/sbin/:$PATH' >> /etc/profile
# tail -2 /etc/profile
# add moosefs to the path variable
PATH=/usr/local/mfs/sbin/:$PATH
# source /etc/profile
#配置开机自启动
# echo '# Configure the metalogger service startup' >> /etc/rc.local
# echo '/usr/local/mfs/sbin/mfsmetalogger start' >> /etc/rc.local
# tail -2 /etc/rc.local
# Configure the metalogger service startup
/usr/local/mfs/sbin/mfsmetalogger start 九、安装mfs-client
mfs的client端需要安装fuse才能使用mfs文件系统
# lsmod |grep fuse #结果显示没有fuse的模块
# wget http://jaist.dl.sourceforge.net/project/fuse/fuse-2.X/2.9.3/fuse-2.9.3.tar.gz
# tar xf fuse-2.9.3.tar.gz
# cd fuse-2.9.3
# ./configure
# make && make install
#调整环境变量
# echo 'export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH' > /etc/profile
# tail -1 /etc/profile
export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH
#添加fuse模块到内核并开机自启动
# lsmod |grep fuse
# modprobe fuse
# lsmod |grep fuse
fuse 692530
# echo 'modeprobe fuse' >> /etc/sysconfig/modules/fuse.modules
# cat /etc/sysconfig/modules/fuse.modules
modeprobe fuse
# chmod 755 /etc/sysconfig/modules/fuse.modules mfs客户端需要安装mfsmount
# yum install zlib-devel fuse-devel -y
# groupadd -g 1000 mfs
# useradd -u 1000 -g mfs mfs -s /sbin/nologin
# tar xf mfs-1.6.27-5.tar.gz
# cd mfs-1.6.27
# ./configure --prefix=/usr/local/mfs-1.6.27 --with-default-user=mfs --with-default-group=mfs --disable-mfsmaster --disable-mfschunkserver --enable-mfsmount
# make && make install
# ln -s /usr/local/mfs-1.6.27 /usr/local/mfs
# ll -d /usr/local/mfs
lrwxrwxrwx 1 root root 21 Jul 23 10:15 /usr/local/mfs -> /usr/local/mfs-1.6.27
# ll /usr/local/mfs/
total 16
drwxr-xr-x 2 root root 4096 Jul 23 10:15 bin
drwxr-xr-x 3 root root 4096 Jul 23 10:15 etc
drwxr-xr-x 2 root root 4096 Jul 23 10:15 sbin
drwxr-xr-x 4 root root 4096 Jul 23 10:15 share
#创建数据目录挂载点
# mkdir /mfsdata
# chown -R mfs.mfs /mfsdata
# ll -d /mfsdata
drwxr-xr-x 2 mfs mfs 4096 Jul 23 10:16 /mfsdata
#挂载前的情况
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.4G 16G 8% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 194M 28M156M16% /boot
#挂载mfs文件系统
#注：这里的密码是上文的mfs-master配置中给定的
# /usr/local/mfs/bin/mfsmount /mfsdata/ -H 192.168.3.35 -o mfspassword=centos
mfsmaster accepted connection with parameters: read-write,restricted_ip,map_all ; root mapped to mfs:mfs ; users mapped to root:root
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.4G 16G 8% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 194M 28M156M16% /boot
192.168.3.35:9421 93G 0 93G 0% /mfsdata #这里显示已挂载上mfs文件系统
#配置环境变量
# echo '# add moosefs to the path variable' >> /etc/profile
# echo 'PATH=/usr/local/mfs/bin/:$PATH' >> /etc/profile
# tail -2 /etc/profile
# add moosefs to the path variable
PATH=/usr/local/mfs/bin/:$PATH
# source /etc/profile
#添加到开机自动挂载
# echo 'Moosefs boot automatically mount' >> /etc/rc.local
# echo '/usr/local/mfs/bin/mfsmount/mfsdata/ -H 192.168.3.75 -o mfspassword=centos' >> /etc/rc.local
# tail -2 /etc/rc.local
Moosefs boot automatically mount
/usr/local/mfs/bin/mfsmount/mfsdata/ -H 192.168.3.75 -o mfspassword=centos 最后测试：当mfs-master宕机后，mfs-client能否正常读写mfs文件系统
#在mfs文件系统中写入测试文件mfs-client1.txt
# touch /mfsdata/mfs-client1.txt
# ll /mfsdata/
total 0
-rw-r--r-- 1 mfs mfs 0 Jul 23 10:22 mfs-client1.txt
#停止mfs-master的heartbeat服务
# /etc/init.d/heartbeat stop
Stopping High-Availability services: Done.
#检查mfs-slave是否将资源接管过来
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.5G 16G 9% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 190M 48M132M27% /boot
/dev/drbd1 15G 41M 14G 1% /usr/local/mfs
# ip a |grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.34/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.35/24 brd 192.168.3.255 scope global secondary eth0
# netstat -lanpt |grep mfs
tcp 0 0 0.0.0.0:9419 0.0.0.0:* LISTEN 4307/mfsmaster
tcp 0 0 0.0.0.0:9420 0.0.0.0:* LISTEN 4307/mfsmaster
tcp 0 0 0.0.0.0:9421 0.0.0.0:* LISTEN 4307/mfsmaster
tcp 0 0 192.168.3.35:9420 192.168.3.36:51820 ESTABLISHED 4307/mfsmaster
tcp 0 0 192.168.3.35:9421 192.168.3.37:44509 ESTABLISHED 4307/mfsmaster
#在客户端查询mfs文件系统
# ll /mfsdata/
total 0
-rw-r--r-- 1 mfs mfs 0 Jul 23 10:22 mfs-client1.txt #可以正常查询
#再创建一个测试文件
# touch /mfsdata/mfs-client2.txt
# ll /mfsdata/
total 0
-rw-r--r-- 1 mfs mfs 0 Jul 23 10:22 mfs-client1.txt
-rw-r--r-- 1 mfs mfs 0 Jul 23 10:24 mfs-client2.txt
#将mfs-master的heartbeat服务重新启动
# /etc/init.d/heartbeat start
Starting High-Availability services: INFO:Resource is stopped
Done.
# df -h
Filesystem SizeUsed Avail Use% Mounted on
/dev/sda3 18G1.6G 16G10% /
tmpfs 242M 0242M 0% /dev/shm
/dev/sda1 190M 48M132M27% /boot
/dev/drbd1 15G 41M 14G 1% /usr/local/mfs
# ip a|grep eth0
2: eth0:mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.33/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.35/24 brd 192.168.3.255 scope global secondary eth0
# netstat -anpt |grep mfs
tcp 0 0 0.0.0.0:9419 0.0.0.0:* LISTEN 4087/mfsmaster
tcp 0 0 0.0.0.0:9420 0.0.0.0:* LISTEN 4087/mfsmaster
tcp 0 0 0.0.0.0:9421 0.0.0.0:* LISTEN 4087/mfsmaster
tcp 0 0 192.168.3.35:9420 192.168.3.36:51822 ESTABLISHED 4087/mfsmaster
tcp 0 0 192.168.3.35:9421 192.168.3.37:44524 ESTABLISHED 4087/mfsmaster
#再次在mfs-client上查看mfs文件系统并写入一个测试文件
# ll /mfsdata/
total 0
-rw-r--r-- 1 mfs mfs 0 Jul 23 10:22 mfs-client1.txt
-rw-r--r-- 1 mfs mfs 0 Jul 23 10:24 mfs-client2.txt
# touch /mfsdata/mfs-client3.txt
# ll /mfsdata/
total 0
-rw-r--r-- 1 mfs mfs 0 Jul 23 10:22 mfs-client1.txt
-rw-r--r-- 1 mfs mfs 0 Jul 23 10:24 mfs-client2.txt
-rw-r--r-- 1 mfs mfs 0 Jul 23 10:26 mfs-client3.txt
#以上整个过程正常，没有任何的卡顿现象发生 到此heartbeat+drbd+mfs的高可用基本完成
页:
[1]