hadoop-2.7.4+hbase-1.3.1+zookeeper-3.4.9搭建分布式集群环境

yyert

# 系统信息
3台系统： centos6.8内核：4   内存：4G  硬盘：50G
# 主机名称，ip地址
master: 192.168.1.110
slave1: 192.168.1.111
slave2: 192.168.1.112

######################## 软件下载地址 ########################
链接：https://pan.baidu.com/s/1dFuBnKt 密码：rhwu
########################   基础初始配置 ########################
# 版本选择
jdk-8u77-linux-x64.rpm
zookeeper-3.4.9.tar.gz
hbase-1.3.1-bin.tar.gz
hadoop-2.7.4.tar.gz

# 配置hosts文件，三台机器都需要
[iyunv@master ~]# cat /etc/hosts
192.168.1.110 master
192.168.1.111 slave1
192.168.1.112 slave2

# 配置用户

1 2	groupadd -g 4000 hadoop useradd -g 4000 -u 4001 hadoop

# 所有的主机 hbase,zookeeper 安装目录都在此处

1 2	mkdir /opt/hadoop chown hadoop.hadoop /opt/hadoop/ -R

########################  时间配置 ########################
# 双机互信
主要有三步：
①生成公钥和私钥
②导入公钥到认证文件
③更改权限

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23

[iyunv@master ~]# ssh-keygen -t rsa Generating public/private rsa key pair. Enter file in which to save the key (/root/.ssh/id_rsa): Enter passphrase (empty for no passphrase): Enter same passphrase again: Your identification has been saved in /root/.ssh/id_rsa. Your public key has been saved in /root/.ssh/id_rsa.pub. The key fingerprint is: ee:15:03:c7:3a:a2:8e:6a:c1:0c:74:d3:97:34:77:04 root@master The key's randomart image is: +--[ RSA 2048]----+ |    . .o.Eoo     | | . o . oo..      | |. . . . . o      | |.        +       | |+     . S o      | | +   . o . o     | |  . .   . .      | | . o   . .       | |o.. .   .        | +-----------------+ [iyunv@master ~]# cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys [iyunv@master ~]# chmod 700 ~/.ssh && chmod 600 ~/.ssh/*

# 主机与从机之间必须可以双向无密码登陆，从机与从机之间无限制

1 2	scp ~/.ssh/authorized_keys slave1:/root/.ssh/ scp ~/.ssh/authorized_keys slave2:/root/.ssh/

# 同步时间

1 2	[iyunv@master zookeeper]# ansible hbase -m cron -a "name='ntpdate' hour='*/1' job='/usr/sbin/ntpdate 192.168.1.110 &> /dev/null'" [iyunv@master zookeeper]# ansible hbase -m shell -a "crontab -l"

#  时间一定要保持一致


########################  防火墙配置 ########################

# 防火墙配置  所有的主机上都得配置，或者开放 (2181，2888:3888端口,这部分端口是zookeeper端口)

1 2 3

[iyunv@slave2 zookeeper]# iptables -I INPUT -s 192.168.1.0/24 -j ACCEPT [iyunv@slave2 zookeeper]# service iptables save [iyunv@slave2 zookeeper]# service iptables restart

########################  JDK配置 ########################

# 安装jdk，并配置环境变量，三台机器都需要安装
# 设置环境变量

1 2 3

[iyunv@slave2 ~]# cat /etc/profile.d/java.sh     export JAVA_HOME=/usr/java/default     export PATH=$JAVA_HOME/bin:$PATH

# 重新加载配置文件使之生效
[iyunv@slave2 ~]# source /etc/profile.d/java.sh

# 查看是否配置完成，3台机器都需要测试
[iyunv@slave2 ~]# java -version
java version "1.8.0_77"
Java(TM) SE Runtime Environment (build 1.8.0_77-b03)
Java HotSpot(TM) 64-Bit Server VM (build 25.77-b03, mixed mode)

######################## zookeeper集群配置 ########################
# 参考文档： http://blog.csdn.net/reblue520/article/details/52279486

# 注意：zookeeper因为有主节点和从节点的关系，所以部署的集群台数最好为奇数个，否则可能出现脑裂导致服务异常

# 下载地址： http://archive.apache.org/dist/z ... keeper-3.4.9.tar.gz
# 注意三台机器都需要安装，如果对ansible熟悉的话 可以直接使用它

1 2	mkdir /opt/hadoop chown hadoop.hadoop /opt/hadoop/ -R

# 安装zookeeper

1 2 3 4

[iyunv@master ~]# cd /opt/hadoop/ [iyunv@master hadoop]# ls zookeeper-3.4.9.tar.gz [iyunv@master hadoop]# tar xf zookeeper-3.4.9.tar.gz

# 弄一个软链接，配置文件直接指向这个地址，未来方便更新版本

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16

[iyunv@master hadoop]# ln -sv zookeeper-3.4.9 zookeeper "zookeeper" -> "zookeeper-3.4.9" [iyunv@master conf]# cd /opt/hadoop/zookeeper/conf [iyunv@master conf]# cp zoo_sample.cfg zoo.cfg [iyunv@master conf]# cat zoo.cfg     tickTime=2000     initLimit=10     syncLimit=5     dataDir=/opt/hadoop/zookeeper/data     dataLogDir=/opt/hadoop/zookeeper/logs     clientPort=2181     server.1=master:2888:3888     server.2=slave1:2888:3888     server.3=slave2:2888:3888

# 创建数据以及日志目录，将设置属主属组权限

1 2	[iyunv@master conf]# mkdir /opt/hadoop/zookeeper/data [iyunv@master conf]# mkdir /opt/hadoop/zookeeper/logs

# 在zoo.cfg中的dataDir指定的目录下，新建myid文件。
# 例如：$ZK_INSTALL/data下，新建myid。在myid文件中输入1。表示为server.1。
echo "1" > data/myid   这里表示的是server.1  如果是第二个机器那么表示server.2

启动：在集群中的每台主机上执行如下命令
bin/zkServer.sh start

查看状态，可以看到其中一台为主节点，其他两台为从节点：
bin/zkServer.sh status

# 启动zookeeper集群

1 2 3 4

[iyunv@master zookeeper]# bin/zkServer.sh status ZooKeeper JMX enabled by default Using config: /opt/hadoop/zookeeper/bin/../conf/zoo.cfg Mode: leader

# 从节点

1 2 3 4

[iyunv@slave1 zookeeper]# bin/zkServer.sh start ZooKeeper JMX enabled by default Using config: /opt/hadoop/zookeeper/bin/../conf/zoo.cfg Starting zookeeper ... STARTED

#  启动报错  说明没有配置myid文件，

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15

2017-12-04 11:56:21,306 [myid:] - INFO  [main:QuorumPeerConfig@124] - Reading configuration from: /opt/hadoop/zookeeper/bin/../conf/zoo.cfg 2017-12-04 11:56:21,323 [myid:] - INFO  [main:QuorumPeer$QuorumServer@149] - Resolved hostname: slave2 to address: slave2/192.168.1.112 2017-12-04 11:56:21,324 [myid:] - INFO  [main:QuorumPeer$QuorumServer@149] - Resolved hostname: slave1 to address: slave1/192.168.1.111 2017-12-04 11:56:21,324 [myid:] - INFO  [main:QuorumPeer$QuorumServer@149] - Resolved hostname: master to address: master/192.168.1.110 2017-12-04 11:56:21,325 [myid:] - INFO  [main:QuorumPeerConfig@352] - Defaulting to majority quorums 2017-12-04 11:56:21,326 [myid:] - ERROR [main:QuorumPeerMain@85] - Invalid config, exiting abnormally org.apache.zookeeper.server.quorum.QuorumPeerConfig$ConfigException: Error processing /opt/hadoop/zookeeper/bin/../conf/zoo.cfg     at org.apache.zookeeper.server.quorum.QuorumPeerConfig.parse(QuorumPeerConfig.java:144)     at org.apache.zookeeper.server.quorum.QuorumPeerMain.initializeAndRun(QuorumPeerMain.java:101)     at org.apache.zookeeper.server.quorum.QuorumPeerMain.main(QuorumPeerMain.java:78) Caused by: java.lang.IllegalArgumentException: /opt/hadoop/zookeeper/data/myid file is missing     at org.apache.zookeeper.server.quorum.QuorumPeerConfig.parseProperties(QuorumPeerConfig.java:362)     at org.apache.zookeeper.server.quorum.QuorumPeerConfig.parse(QuorumPeerConfig.java:140)     ... 2 more Invalid config, exiting abnormally

# 这里是因为防火墙开着，没有开放端口的原因

1 2 3 4 5

2016-03-26 03:48:07,957 [myid:1] - WARN [QuorumPeer[myid=1]/0:0:0:0:0:0:0:0:2181:QuorumCnxManager@400] - Cannot open channel to 3 at election address S2/这里是地址 java.net.ConnectException: 主机无法连接 at java.net.PlainSocketImpl.socketConnect(Native Method) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)

########################   hbase 与hadoop的版本需要对应 ########################
  http://blog.csdn.net/shuaigexiaobo/article/details/78114221    低版本与高版本会安不上，还需要注意jdk版本

######################## hadoop 集群配置 ########################

# 软件放置路径为初级配置的路径 /opt/hadoop

1 2 3

[iyunv@master hadoop]# tar xf hadoop-2.7.4.tar.gz [iyunv@master hadoop]# ln -sv hadoop-2.7.4 hadoop "hadoop" -> "hadoop-2.7.4"

# 配置属主属组权限

1	[iyunv@master hadoop]# chown hadoop.hadoop /opt/hadoop/hadoop-2.7.4 -R

# 环境变量设置

1 2 3 4 5 6 7 8 9 10

vim /etc/profile.d/hadoop.sh export HADOOP_HOME=/opt/hadoop/hadoop export HADOOP_INSTALL=$HADOOP_HOME   export HADOOP_MAPRED_HOME=$HADOOP_HOME   export HADOOP_COMMON_HOME=$HADOOP_HOME   export HADOOP_HDFS_HOME=$HADOOP_HOME   export YARN_HOME=$HADOOP_HOME   export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native   export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin   # export HADOOP_SSH_OPTS="-p 22"

# 复制到其它主机中

1 2	[iyunv@master hadoop]# scp /etc/profile.d/hadoop.sh slave1:/etc/profile.d/ [iyunv@master hadoop]# scp /etc/profile.d/hadoop.sh slave2:/etc/profile.d/

# 加载环境变量

1	[iyunv@master hadoop]# soure /etc/profile.d/hadoop.sh

# 查看是否生效

1 2 3

[iyunv@master hadoop]# hadoop version Hadoop 2.7.4 Subversion https://shv@git-wip-us.apache.org/repos/asf/hadoop.git -r cd915e1e8d9d0131462a0b7301586c175728a282

# hadoop配置文件在放置于/opt/hadoop/hadoop/etc/hadoop

1 2 3 4 5 6 7

vim  core-site.xml    # 添加如下内容 <configuration>     <property>         <name>fs.default.name</name>         <value>hdfs://master:9000</value>     </property> </configuration>

1 2 3

vim hadoop-env.sh #export JAVA_HOME=${JAVA_HOME} export JAVA_HOME=/usr/java/default

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18

vim hdfs-site.xml   # 配置hdfs文件数据节点以及名称节点 <configuration>    <property>         <name>dfs.name.dir</name>         <value>/opt/hadoop/hadoop/name</value>     </property>     <property>         <name>dfs.data.dir</name>         <value>/opt/hadoop/hadoop/data</value>     </property>     <property>         <name>dfs.replication</name>         <value>3</value>     </property> </configuration> mkdir /opt/hadoop/hadoop/name mkdir /opt/hadoop/hadoop/data

1 2 3 4 5 6 7 8

[iyunv@master hadoop]# cp mapred-site.xml.template mapred-site.xml [iyunv@master hadoop]# vim !$ <configuration>     <property>         <name>mapred.job.tracker</name>         <value>master:9001</value>     </property> </configuration>

# 配置从节点 先删除localhost

1 2 3

/opt/hadoop/hadoop/etc/hadoop/slaves slave1 slave2

# 三台机器都是一样的配置，放置相同的路径

1 2	[iyunv@master hadoop]# scp -r hadoop-2.7.4 slave1:/opt/hadoop/ [iyunv@master hadoop]# scp -r hadoop-2.7.4 slave2:/opt/hadoop/

# 使用ansible或者手动直接软链接过去就行

1	[iyunv@master hadoop]# ansible hbase -m shell -a 'ln -sv /opt/hadoop/hadoop-2.7.4 /opt/hadoop/hadoop'

# 配置属主属组文件

1	[iyunv@master hadoop]# ansible hbase -m shell -a 'chown hadoop.hadoop /opt/hadoop/hadoop -R'

# 进入master的/opt/hadoop/hadoop目录，执行以下操作

1	# bin/hadoop namenode -format                # 格式化namenode，第一次启动服务前执行的操作,以后不需要执行

# 启动hadoop服务

1 2 3 4 5

[iyunv@master logs]# sbin/start-all.sh This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh 17/12/04 15:56:51 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Starting namenodes on [master] master: starting namenode, logging to /opt/hadoop/hadoop-2.7.4/logs/hadoop-root-namenode-master.out

# 查看进程   会发现多了资源名称节点以及namanode

1 2 3 4 5 6

[iyunv@master logs]# jps 5057 ResourceManager 4900 SecondaryNameNode 4709 NameNode 5208 Jps 2734 QuorumPeerMain

# 登陆其它节点  会发现多了一个数据节点

1 2 3 4 5

[iyunv@slave2 hadoop]# jps    2624 QuorumPeerMain 3489 NodeManager 3378 DataNode 3603 Jps

######################## hbase集群配置 ########################
# 软件放置路径为初级配置的路径 /opt/hadoop

1 2 3

[iyunv@master hadoop]# tar xf hbase-1.3.1-bin.tar.gz [iyunv@master hadoop]# ln -sv hbase-1.3.1 hbase "hbase" -> "hbase-1.3.1"

# 配置文件目录 /opt/hadoop/hbase/conf
vim hbase-env.sh

1 2 3

export JAVA_HOME=/usr/java/default/ export HBASE_CLASSPATH=/opt/hadoop/hadoop/etc/hadoop export HBASE_MANAGES_ZK=false          # 不使用自带的zk，使用独立的zookeeper

vim hbase-site.xml    # 配置站点信息

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30

<configuration>     <property>         <name>hbase.rootdir</name>         <value>hdfs://master:9000/hbase</value>     </property>     <property>         <name>hbase.master</name>         <value>master</value>     </property>     <property>         <name>hbase.cluster.distributed</name>         <value>true</value>     </property>     <property>         <name>hbase.zookeeper.property.clientPort</name>         <value>2181</value>                                     # 这里指的是zook的端口     </property>     <property>         <name>hbase.zookeeper.quorum</name>                     # 主机名一定要对应上         <value>master,slave1,slave2</value>     </property>     <property>         <name>zookeeper.session.timeout</name>                  # zook的session超时时长         <value>60000000</value>     </property>     <property>         <name>dfs.support.append</name>         <value>true</value>     </property> </configuration>

vim regionservers        # 配置从节点 一定要对应上

1 2	slave1 slave2

# 设置软链接，方便未来升级
[iyunv@master hadoop]# ansible hbase -m shell -a "ln -sv /opt/hadoop/hbase-1.3.1 /opt/hadoop/hbase"
# 设置属主属组权限
[iyunv@master hadoop]# ansible hbase -m shell -a "chown hadoop.hadoop /opt/hadoop/hbase-1.3.1 -R"

# 启动三台机器上的 hbase服务
[iyunv@master hadoop]# ansible hbase -m shell -a "/opt/hadoop/hbase-1.3.1/bin/start-hbase.sh"


# 只需要启动master上的，其它机器上会自动启动
[iyunv@master hadoop]# /opt/hadoop/hbase/bin/start-hbase.sh
# 查看master上的服务

1 2 3 4 5 6 7

[iyunv@master hadoop]# jps 5057 ResourceManager 4900 SecondaryNameNode 6516 HMaster 4709 NameNode 6809 Jps 2734 QuorumPeerMain

# 查看slave上的从节点服务

1 2 3 4 5 6 7

[iyunv@slave1 ~]# jps 3510 NodeManager 3399 DataNode 2680 QuorumPeerMain 5464 Jps 5049 HMaster 4730 HRegionServer

# 进入hbase shell进行验证
/opt/hadoop/hbase/bin/hbase shell

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26

2017-12-04 16:20:28,690 WARN  [main] util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable SLF4J: Class path contains multiple SLF4J bindings. SLF4J: Found binding in [jar:file:/opt/hadoop/hbase-1.3.1/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/opt/hadoop/hadoop-2.7.4/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] HBase Shell; enter 'help<RETURN>' for list of supported commands. Type "exit<RETURN>" to leave the HBase Shell Version 1.3.1, r930b9a55528fe45d8edce7af42fef2d35e77677a, Thu Apr  6 19:36:54 PDT 2017 hbase(main):001:0> hbase(main):002:0* list TABLE                                                                                                                                                0 row(s) in 0.2350 seconds => [] hbase(main):003:0> create 'scores', 'grade', 'course' 0 row(s) in 2.4310 seconds => Hbase::Table - scores hbase(main):004:0> list TABLE                                                                                                                                                scores                                                                                                                                              1 row(s) in 0.0080 seconds => ["scores"]

####  此处打开的地址都是 master 的IP   ，  192.168.1.110