Glusterfs分布式存储的安装

桀昊j

昨日小测了Glusterfs分布式存储系统，此次采用了官方最新版的glusterfs-3.0.2.tar.gz 具体安装配置如下：

一、操作系统和主机信息:
采用CentOS 5.2操作系统
client 192.168.1.9
Srv01 192.168.1.10
Srv02 192.168.1.11

客户和服务器端安装如下包：
rpm -ivh http://vault.centos.org/5.2/centosplus/x86_64/RPMS/kernel-2.6.18-92.1.6.el5.centos.plus.x86_64.rpm
rpm -ivh http://vault.centos.org/5.2/centosplus/x86_64/RPMS/kernel-devel-2.6.18-92.1.6.el5.centos.plus.x86_64.rpm
rpm -ivh http://vault.centos.org/5.2/centosplus/x86_64/RPMS/kernel-headers-2.6.18-92.1.6.el5.centos.plus.x86_64.rpm

二、客户端(client)配置:
yum install libibverbs-devel fuse-devel fuse fuse-libs
wget http://ftp.gluster.com/pub/gluster/glusterfs/3.0/LATEST/glusterfs-3.0.2.tar.gz
tar zxvf glusterfs-3.0.2.tar.gz
cd glusterfs-3.0.2
./configure --enable-fusermount
make && make install
ldconfig

验证:
glusterfs --version
mkdir -p /mnt/glusterfs

cat  /usr/local/etc/glusterfs/client-stor.vol
## file auto generated by /usr/local/bin/glusterfs-volgen (mount.vol)
# Cmd line:
# $ /usr/local/bin/glusterfs-volgen --name repstore --raid 1 Srv01:/glusterfs/b Srv02:/glusterfs/b Srv01:/glusterfs/c Srv02:/glusterfs/c Srv01:/glusterfs/d Srv02:/glusterfs/

# RAID 1
# TRANSPORT-TYPE tcp
volume Srv01-1
        type protocol/client
        option transport-type tcp/client
        option remote-host 192.168.1.10
        option remote-port 6996
        option remote-subvolume brick1
end-volume

volume Srv01-2
        type protocol/client
        option transport-type tcp/client
        option remote-host 192.168.1.10
        option remote-port 6996
        option remote-subvolume brick2
end-volume

volume Srv01-3
        type protocol/client
        option transport-type tcp/client
        option remote-host 192.168.1.10
        option remote-port 6996
        option remote-subvolume brick3
end-volume

volume Srv02-1
        type protocol/client
        option transport-type tcp/client
        option remote-host 192.168.1.11
        option remote-port 6996
        option remote-subvolume brick1
end-volume

volume Srv02-2
        type protocol/client
        option transport-type tcp/client
        option remote-host 192.168.1.11
        option remote-port 6996
        option remote-subvolume brick2
end-volume

volume Srv02-3
        type protocol/client
        option transport-type tcp/client
        option remote-host 192.168.1.11
        option remote-port 6996
        option remote-subvolume brick3
end-volume

volume mirror-0
        type cluster/replicate
        subvolumes Srv01-1 Srv02-1
end-volume

volume mirror-1
        type cluster/replicate
        subvolumes Srv01-2 Srv02-2
end-volume

volume mirror-2
        type cluster/replicate
        subvolumes Srv01-3 Srv02-3
end-volume

volume unify
        type cluster/distribute
        subvolumes mirror-0 mirror-1 mirror-2
end-volume

volume writebehind
        type performance/write-behind
        option cache-size 4MB
        subvolumes unify
end-volume

volume readahead
        type performance/read-ahead
        option page-count 4
        subvolumes writebehind
end-volume

volume iocache
        type performance/io-cache
        option cache-size `grep 'MemTotal' /proc/meminfo    | awk '{print $2 * 0.2 / 1024}' | cut -f1 -d.`MB
        option cache-timeout 1
        subvolumes readahead
end-volume

volume quickread
        type performance/quick-read
        option cache-timeout 1
        option max-file-size 64kB
        subvolumes iocache
end-volume

volume statprefetch
        type performance/stat-prefetch
        subvolumes quickread
end-volume

volume mountpoint1
        type mount/fuse
        option mountpoint /mnt/glusterfs
        subvolumes statprefetch
end-volume
EOF


三、服务器端(Srv01/Srv02)配置:
rpm -ivh ftp://ftp.pbone.net/mirror/ftp.centos.org/5.4/centosplus/x86_64/RPMS/reiserfs-utils-3.6.19-2.4.1.x86_64.rpm
mkdir -p /glusterfs/{b,c,d}
mkfs.reiserfs -f /dev/sdb
mkfs.reiserfs -f /dev/sdc
mkfs.reiserfs -f /dev/sdd
wget http://ftp.gluster.com/pub/gluster/glusterfs/3.0/LATEST/glusterfs-3.0.2.tar.gz
tar zxvf glusterfs-3.0.2.tar.gz
cd glusterfs-3.0.2
./configure --enable-fusermount
make && make install
ldconfig

验证:
glusterfs --version

cat  /usr/local/etc/glusterfs/server-export.vol
## file auto generated by /usr/local/bin/glusterfs-volgen (export.vol)
# Cmd line:
# $ /usr/local/bin/glusterfs-volgen --name repstore --raid 1 Srv01:/glusterfs/b Srv02:/glusterfs/b Srv01:/glusterfs/c Srv02:/glusterfs/c Srv01:/glusterfs/d Srv02:/glusterfs/

volume posix1
    type storage/posix
    option directory /glusterfs/b
end-volume

volume locks1
        type features/locks
        subvolumes posix1
end-volume

volume brick1
        type performance/io-threads
        option thread-count 8
        subvolumes locks1
end-volume

volume posix2
    type storage/posix
    option directory /glusterfs/c
end-volume

volume locks2
        type features/locks
        subvolumes posix2
end-volume

volume brick2
        type performance/io-threads
        option thread-count 8
        subvolumes locks2
end-volume

volume posix3
    type storage/posix
    option directory /glusterfs/d
end-volume

volume locks3
        type features/locks
        subvolumes posix3
end-volume

volume brick3
        type performance/io-threads
        option thread-count 8
        subvolumes locks3
end-volume

volume server-tcp
        type protocol/server
        option transport-type tcp/server
        option auth.addr.brick1.allow *
        option auth.addr.brick2.allow *
        option auth.addr.brick3.allow *
        option transport.socket.listen-port 6996
        option transport.socket.nodelay on
        subvolumes brick1 brick2 brick3
end-volume
EOF


四、启动顺序:
1、服务器端(Srv01/Srv02)
glusterfs -l /usr/local/etc/glusterfs/server-exportlog -f /usr/local/etc/glusterfs/server-export.vol
2、客户端(client)
glusterfs -l /usr/local/etc/glusterfs/client-stor.log -f /usr/local/etc/glusterfs/client-stor.vol

3、客户端验证(client)
# df -h
Filesystem            Size  Used Avail Use% Mounted on
/dev/mapper/VolGroup00-LogVol00
                       71G  2.3G   65G   4% /
/dev/sda1              99M   19M   76M  20% /boot
tmpfs                 502M     0  502M   0% /dev/shm
glusterfs#/usr/local/etc/glusterfs/client-stor.vol
                      1.4T  1.3G  1.4T   1% /mnt/glusterfs