CentOS 6.5 编译Eclipse plugin for hadoop 2.5及加载连接
1. rpm -ivh jdk-7u67-linux-x64.rpmcd /opt
tar zxf /root/hadoop-2.2.0.tar.gz
tar zxf /root/apache-ant-1.9.4-bin.tar.gz
tar zxf /root/apache-maven-3.2.3-bin.tar.gz
tar zxf /root/eclipse-java-luna-SR1-linux-gtk-x86_64.tar.gz
unzip -x /root/hadoop2x-eclipse-plugin-master.zip (https://github.com/winghc/hadoop2x-eclipse-plugin)
ln -s apache-ant-1.9.4 ant
ln -s apache-maven-3.2.3 maven
ln -s hadoop-2.2.0 hadoop
2. echo 'export JAVA_HOME=/usr/java/default' >> /etc/profile
echo 'export HADOOP_HOME=/opt/hadoop' >> /etc/profile
echo 'export ANT_HOME=/opt/ant' >> /etc/profile
echo 'export MAVEN_HOME=/opt/maven' >> /etc/profile
echo 'export PATH=$JAVA_HOME/bin:$ANT_HOME/bin:$MAVEN_HOME/bin:$HADOOP_HOME/bin:$PATH' >> /etc/profile
source /etc/profile
3. cp /root/ivy-2.1.0.jar /opt/hadoop2x-eclipse-plugin-master/ivy/
cd /opt/hadoop2x-eclipse-plugin-master/src/contrib/eclipse-plugin
ant jar -Dversion=2.5.0 -Declipse.home=/opt/eclipse -Dhadoop.home=/opt/hadoop
cp /opt/hadoop2x-eclipse-plugin-master/build/contrib/eclipse-plugin/hadoop-eclipse-plugin-2.5.0.jar /opt/eclipse/plugins/
4. run eclipse: /opt/eclipse/eclipse (此目录权限最好修改为：chmod -R 755 /opt/eclipse; chown -R hdfs:hadoop /opt/eclipse)
连接hadoop环境为：CDH 5.2 on a Single Linux Node in Pseudo-distributed Mode（安装参考附录）
5. Windows 7一样可使用这个hadoop-eclipse-plugin-2.5.0.jar
附录：
Installing CDH 5 with YARN on a Single Linux Node in Pseudo-distributed mode
分区考虑，不要使用LVM
root -- >50G
opt -- >50G
swap -- 2倍系统内存
RAM -- >8GB
1. disable selinux and iptables
service iptables stop
chkconfig iptables off; chkconfig ip6tables off
setenforce 0
sed -i 's,SELINUX=enforcing,SELINUX=disabled,g' /etc/selinux/config
2. disable ipv6 and kernel parameters tuning
echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf
echo "vm.swappiness = 0" >> /etc/sysctl.conf
echo 'net.ipv4.tcp_retries2 = 2' >> /etc/sysctl.conf
echo 'vm.overcommit_memory = 1' >> /etc/sysctl.conf
echo "fs.file-max = 6815744" >> /etc/sysctl.conf
echo "fs.aio-max-nr = 1048576" >> /etc/sysctl.conf
echo "net.core.rmem_default = 262144" >> /etc/sysctl.conf
echo "net.core.wmem_default = 262144" >> /etc/sysctl.conf
echo "net.core.rmem_max = 16777216" >> /etc/sysctl.conf
echo "net.core.wmem_max = 16777216" >> /etc/sysctl.conf
echo "net.ipv4.tcp_rmem = 4096 262144 16777216" >> /etc/sysctl.conf
echo "net.ipv4.tcp_wmem = 4096 262144 16777216" >> /etc/sysctl.conf
only on ResourceManager and JobHistory Server
echo "net.core.somaxconn = 1000" >> /etc/sysctl.conf
sysctl -p
echo "echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled" >> /etc/rc.local
echo "echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag" >> /etc/rc.local
echo "echo no > /sys/kernel/mm/redhat_transparent_hugepage/khugepaged/defrag" >> /etc/rc.local
3. vi /etc/hosts to add all hosts FQDN, like below:
192.168.1.10 test.gkytech.com test
vi /etc/sysconfig/network to set FQDN
cat/etc/yum.repos.d/iso.repo
name=iso
baseurl=http://192.168.1.10/centos65
enable=1
gpgcheck=0
EOF
yum -y install ntp openssh-clients lzo
service ntpd start; chkconfig ntpd on
vi /etc/security/limits.conf
* soft core unlimited
* hard core unlimited
* soft nofile 65536
* hard nofile 65536
* soft nproc unlimited
* hard nproc unlimited
* soft memlock unlimited
* hard memlock unlimited
vi /etc/grub.conf
add "elevator=deadline"(no quotes) at the end of kernel line
reboot to take effect
4. cat/etc/yum.repos.d/cdh520.repo
name=cm520
baseurl=http://192.168.1.10/cdh520
enable=1
gpgcheck=0
EOF
rpm -ivh jdk-7u67-linux-x64.rpm
echo 'export JAVA_HOME=/usr/java/default' >> /etc/profile
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile
source /etc/profile
# To install Hadoop with YARN
yum -y install hadoop-conf-pseudo
# Step 1: Format the NameNode
sudo -u hdfs hdfs namenode -format
# Step 2: Start HDFS
for x in `cd /etc/init.d ; ls hadoop-hdfs-*` ; do service $x start ; done
# Step 3: Create the directories needed for Hadoop processes
/usr/lib/hadoop/libexec/init-hdfs.sh
# Step 4: Verify the HDFS File Structure
sudo -u hdfs hadoop fs -ls -R /
# Step 5: Start YARN
optional: to replace localhost with ip address, so you can access web ui
vi /etc/hadoop/conf/core-site.xml
vi /etc/hadoop/conf/mapred-site.xml
service hadoop-yarn-resourcemanager start
service hadoop-yarn-nodemanager start
service hadoop-mapreduce-historyserver start
# Step 6: Create User Directories
useradd test
sudo -u hdfs hadoop fs -mkdir /user/test
sudo -u hdfs hadoop fs -chown test /user/test
# Step 7: Testing
su - test
hadoop jar /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar pi 10 100
页:
[1]