Slave1
1.mountslave1@slave1-VirtualBox:~$ sudo mkdir /mnt/shared
password for slave1:
mkdir: cannot create directory ‘/mnt/shared’: File exists
slave1@slave1-VirtualBox:~$ sudo mount -t vboxsf slave1 /mnt/shared/
slave1@slave1-VirtualBox:~$ ls /mnt/shared/
2.jdk
sudo mkdir /usr/local/jvm
sudo cp /mnt/shared/jdk-7u79-linux-x64.tar.gz /usr/local/jvm/
cd /usr/local/jvm/
slave1@slave1-VirtualBox:/usr/local/jvm$ sudo tar zxf jdk-7u79-linux-x64.tar.gz
slave1@slave1-VirtualBox:/usr/local/jvm$ cd ~
slave1@slave1-VirtualBox:~$ sudo gedit .bashrc
export JAVA_HOME=/usr/local/jvm/jdk1.7.0_79
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH
source .bashrc
lave1@slave1-VirtualBox:~$ sudo update-alternatives --install /usr/bin/java java /usr/local/jvm/jdk1.7.0_79/bin/java 300
update-alternatives: using /usr/local/jvm/jdk1.7.0_79/bin/java to provide /usr/bin/java (java) in auto mode
slave1@slave1-VirtualBox:~$ sudo update-alternatives --install /usr/bin/javac javac /usr/local/jvm/jdk1.7.0_79/bin/javac 300
slave1@slave1-VirtualBox:~$ sudo update-alternatives --install /usr/bin/jar jar /usr/local/jvm/jdk1.7.0_79/bin/jar 300
slave1@slave1-VirtualBox:~$ sudo update-alternatives --install /usr/bin/jar jar /usr/local/jvm/jdk1.7.0_79/bin/jar 300
slave1@slave1-VirtualBox:~$ java -version
3.SSH
sudo apt-get install ssh
slave1@slave1-VirtualBox:~$ ps -e |grep ssh
slave1@slave1-VirtualBox:~$ ssh-keygen -t rsa -P ""
slave2@slave2-VirtualBox:~$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
slave1@slave1-VirtualBox:~$ ssh localhost
slave1@slave1-VirtualBox:~$ exit
4.rsync
slave1@slave1-VirtualBox:~$ sudo apt-get install rsync
5.Hadoop
slave1@slave1-VirtualBox:~$ sudo cp /mnt/shared/hadoop-2.6.0.tar.gz /usr/local/
slave1@slave1-VirtualBox:~$ cd /usr/local/
slave1@slave1-VirtualBox:/usr/local$ sudo tar zxf hadoop-2.6.0.tar.gz
lave1@slave1-VirtualBox:/usr/local$ cd hadoop-2.6.0/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ cd etc/hadoop/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/etc/hadoop$ sudo gedit hadoop-env.sh
export JAVA_HOME=/usr/local/jvm/jdk1.7.0_79
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/etc/hadoop$ cd ~
slave1@slave1-VirtualBox:~$ sudo gedit .bashrc
export PATH=${JAVA_HOME}/bin:/usr/local/hadoop-2.6.0/bin:$PATH
slave1@slave1-VirtualBox:~$ source .bashrc
slave1@slave1-VirtualBox:~$ hadoop version
slave1@slave1-VirtualBox:~$ cd /usr/local/hadoop-2.6.0/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo mkdir input
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo cp README.txt input/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.0.jar wordcount input output
error:
java.lang.Exception: java.io.IOException: Mkdirs failed to create file:/usr/local/hadoop-2.6.0/output/_temporary/0/_temporary/attempt_local2127094074_0001_r_000000_0 (exists=false, cwd=file:/usr/local/hadoop-2.6.0)
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ cd ..
slave1@slave1-VirtualBox:/usr/local$ sudo chmod -R 777 hadoop-2.6.0
slave1@slave1-VirtualBox:/usr/local$ sudo chown -R slave1:adm hadoop-2.6.0
slave1@slave1-VirtualBox:/usr/local$ cd hadoop-2.6.0/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.0.jar wordcount input output
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ cat output/*
6.pre-sudo distribute
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ cd etc/hadoop/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/etc/hadoop$ sudo gedit core-site.xml
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/hadoop-2.6.0/tmp</value>
<description>Abase for other temporary directories.</description>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://localhost:9000</value>
</property>
</configuration>
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/etc/hadoop$ sudo gedit hdfs-site.xml
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/usr/local/hadoop-2.6.0/dfs/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/usr/local/hadoop-2.6.0/dfs/data</value>
</property>
<property> //这个属性节点是为了防止后面eclopse存在拒绝读写设置的
<name>dfs.permissions</name>
<value>false</value>
</property>
</configuration>
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/etc/hadoop$ cd ../..
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo mkdir tmp
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo mkdir dfs dfs/data dfs/name
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo chmod -R 777 dfs/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo chmod -R 777 tmp/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hdfs namenode -format
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sbin/start-all.sh
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ jps
5007 ResourceManager
4591 NameNode
5123 NodeManager
5157 Jps
4873 SecondaryNameNode
no datanode
so
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sbin/stop-all.sh
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo chmod -R 777 dfs/data/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sudo rm -rf tmp/*
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hdfs namenode -format
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sbin/start-all.sh
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sbin/stop-all.sh
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ cd /dfs
bash: cd: /dfs: No such file or directory
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ cd dfs/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/dfs$ sudo chown -R slave1:adm data/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/dfs$ sbin/start-all.sh
bash: sbin/start-all.sh: No such file or directory
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0/dfs$ cd ..
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ sbin/start-all.sh
NameNode - http://localhost:50070/
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hdfs dfs -mkdir /user
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hdfs dfs -mkdir /user/slave1
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hdfs dfs -mkdir /user/slave1/input
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hdfs dfs -put README.txt input
slave1@slave1-VirtualBox:/usr/local/hadoop-2.6.0$ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.0.jar wordcount input output
Examine the output files:
Copy the output files from the distributed filesystem to the local filesystem and examine them:
$ bin/hdfs dfs -get output output
$ cat output/*
or
View the output files on the distributed filesystem:
$ bin/hdfs dfs -cat output/*
When you're done, stop the daemons with:
$ sbin/stop-dfs.sh
页:
[1]