useradd -u 900 hadoop
echo westos | passwd --stdin hadoop
server2:
sh jdk-6u32-linux-x64.bin ##安装JDK
mv jdk1.6.0_32/ /home/hadoop/java
mv hadoop-1.2.1.tar.gz /home/hadoop/
su - hadoop
vim .bash_profile
export JAVA_HOME=/home/hadoop/java
export CLASSPATH=.:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
export PATH=$PATH:$HOME/bin:$JAVA_HOME/bin
source .bash_profile
tar zxf hadoop-1.1.2.tar.gz ##配置hadoop单节点
ln -s hadoop-1.1.2 hadoop
cd /home/hadoop/hadoop/conf
vim hadoop-env.sh
export JAVA_HOME=/home/hadoop/java cd ..
mkdir input
cp conf/*.xml input/
bin/hadoop jar hadoop-examples-1.1.2.jar
bin/hadoop jar hadoop-examples-1.1.2.jar grep input output 'dfs[a-z.]+'
cd output/
cat *
1 dfsadmin 设置master到slave端无密码登陆:
server2:
su - hadoop
ssh-keygen
ssh-copy-id localhost
ssh-copy-id 172.25.45.3
ssh-copy-id 172.25.45.4
cd /home/hadoop/hadoop/conf
vim core-site.xml ##指定 namenode
fs.default.name
hdfs://172.25.45.2:9000
vim mapred-site.xml ##指定 jobtracker
mapred.job.tracker
172.25.45.2:9001
vim hdfs-site.xml ##指定文件保存的副本数
dfs.replication
1
cd ..
bin/hadoop namenode -format ##格式化成一个新的文件系统
ls /tmp
hadoop-hadoop hsperfdata_hadoop hsperfdata_root yum.log
bin/start-dfs.sh ##启动hadoop进程
jps
bin/start-mapred.sh
jps
在浏览器中打开:172.25.45.2:50030
打开172.25.45.2:50070
bin/hadoop fs -put input test ##给分布式文件系统考入新建的文件
bin/hadoop jar hadoop-examples-1.2.1.jar wordcount output
同时在网页中
查看网页中上传的文件:
bin/hadoop fs -get output test
cat test/*
rm -fr test/ ##删除下载的文件
2. server2:
共享文件系统:
su - root
yum install nfs-utils -y
/etc/init.d/rpcbind start
/etc/init.d/nfs start
vim /etc/exports
/home/hadoop *(rw,anonuid=900,anongid=900) exportfs -rv
exportfs -v
server3和server4:
yum install nfs-utils -y
/etc/init.d/rpcbind start
showmount -e 172.25.45.2 ##
Export list for 172.25.45.2:
/home/hadoop *
mount 172.25.45.2:/home/hadoop /home/hadoop/
df
server2:
su - hadoop
cd hadoop/conf
vim hdfs-site.xml
dfs.replication
2
vim slaves ##slave端的ip
172.25.45.3
172.25.45.4
vim masters ##master端的ip
172.25.45.2 提示:##如果还有之前的进程开着,必须先关闭,才能再进行格式化,保证jps没有什么进程运行
关闭进程的步骤
bin/stop-all.sh ##执行完之后,有时tasktracker,datanode会开着,所以要关闭它们
bin/hadoop-daemon.sh stop tasktracker
bin/hadoop-daemon.sh stop datanode
以hadoop用户的身份删除/tmp里的文件,没有权限的文件就留着
su - hadoop
bin/hadoop namenode -format
bin/start-dfs.sh
bin/start-mapred.s
bin/hadoop fs -put input test ##
bin/hadoop jar hadoop-examples-1.2.1.jar grep test output 'dfs[a-z.]+' ##
一边上传一边在浏览器中打开172.25.45.2:50030中观察会发现有正在上传的文件
su - hadoop
bin/hadoop dfsadmin -report
dd if=/dev/zero of=bigfile bs=1M count=200
bin/hadoop fs -put bigfile test
在浏览器中打开172.25.45.2:50070
3.新加server5.example.com 172.25.45.5 作为新的slave端:
su - hadoop
yum install nfs-utils -y
/etc/init.d/rpcbind start
useradd -u 900 hadoop
echo westos | passwd --stdin hadoop
mount 172.25.45.2:/home/hadoop/ /home/hadoop/
su - hadoop
vim hadoop/conf/slaves
172.25.45.3
172.25.45.4
172.25.45.5
cd /home/hadoop/hadoop
bin/hadoop-daemon.sh start datanode
bin/hadoop-daemon.sh start tasktracker
jps
删除一个slave端:
server2:
su - hadoop
cd /home/hadoop/hadoop/conf
vim mapred-site.xml
dfs.hosts.exclude
/home/hadoop/hadoop/conf/datanode-excludes
vim /home/hadoop/hadoop/conf/datanode-excludes
172.25.45.3 ##删除172.25.45.3不作为slave端 cd /home/hadoop/hadoop
bin/hadoop dfsadmin -refreshNodes ##刷新节点
bin/hadoop dfsadmin -report ##查看节点状态,会发现server3上的数据转移到serve5上
在server3上:
su - hadoop
bin/stop-all.sh
cd /home/hadoop/hadoop
bin/hadoop-daemon.sh stop tasktracker
bin/hadoop-daemon.sh stop datanode
server2:
vim /home/hadoop/hadoop/conf/slaves
172.25.45.4
172.25.45.5
4. 配置新版的hadoop:
server2:
su - hadoop
cd /home/hadoop
tar zxf jdk-7u79-linux-x64.tar.gz
ln -s jdk1.7.0_79/ java
tar zxf hadoop-2.6.4.tar.gz
ln -s hadoop-2.6.4 hadoop
cd /home/hadoop/hadoop/etc/hadoop
vim hadoop-env.sh
export JAVA_HOME=/home/hadoop/java
export HADOOP PREFIX=/home/hadoop/hadoop
cd /home/hadoop/hadoop
mkdir inp
cp etc/hadoop/*.xml input
tar -tf hadoop-native-64-2.6.0.tar
tar -xf hadoop-native-64-2.6.0.tar -C hadoop/lib/native/
cd /home/hadoop/hadoop
rm -fr output/
bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.4.jar grep input output 'dfs[a-z.]+'
cd /hone/hadoop/hadoop/etc/hadoop/
vim slaves
172.25.45.3
172.25.45.4
vim core-site.xm
fs.defaultFS
hdfs://172.25.45.2:9000
vim mapred-site.xml
mapred.job.tracker
172.25.45.2:9001
vim hdfs-site.xml
dfs.replication
2
cd /home/hadoop/hadoop
bin/hdfs namenode -format
sbin/start-dfs.sh
jps
bin/hdfs dfs -mkdir /user/hadoop ##要上传的文件,必须在上传之前新建出其目录
bin/hdfs dfs -put input/ test
rm -fr input/
bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.4.jar grep test output 'dfs[a-z.]+'
bin/hdfs dfs -cat output/*
1dfsadmin
在浏览器中打开172.25.45.2:50070