CentOS 6.5 编译Eclipse plugin for hadoop 2.5及加载连接

qq591577286

　　1. rpm -ivh jdk-7u67-linux-x64.rpm
　　cd /opt
　　tar zxf /root/hadoop-2.2.0.tar.gz
　　tar zxf /root/apache-ant-1.9.4-bin.tar.gz
　　tar zxf /root/apache-maven-3.2.3-bin.tar.gz
　　tar zxf /root/eclipse-java-luna-SR1-linux-gtk-x86_64.tar.gz
　　unzip -x /root/hadoop2x-eclipse-plugin-master.zip (https://github.com/winghc/hadoop2x-eclipse-plugin)
　　ln -s apache-ant-1.9.4 ant
　　ln -s apache-maven-3.2.3 maven
　　ln -s hadoop-2.2.0 hadoop
　　2. echo 'export JAVA_HOME=/usr/java/default' >> /etc/profile
　　echo 'export HADOOP_HOME=/opt/hadoop' >> /etc/profile
　　echo 'export ANT_HOME=/opt/ant' >> /etc/profile
　　echo 'export MAVEN_HOME=/opt/maven' >> /etc/profile
　　echo 'export PATH=$JAVA_HOME/bin:$ANT_HOME/bin:$MAVEN_HOME/bin:$HADOOP_HOME/bin:$PATH' >> /etc/profile
　　source /etc/profile
　　3. cp /root/ivy-2.1.0.jar /opt/hadoop2x-eclipse-plugin-master/ivy/
　　cd /opt/hadoop2x-eclipse-plugin-master/src/contrib/eclipse-plugin
　　ant jar -Dversion=2.5.0 -Declipse.home=/opt/eclipse -Dhadoop.home=/opt/hadoop
　　cp /opt/hadoop2x-eclipse-plugin-master/build/contrib/eclipse-plugin/hadoop-eclipse-plugin-2.5.0.jar /opt/eclipse/plugins/
　　4. run eclipse: /opt/eclipse/eclipse (此目录权限最好修改为：chmod -R 755 /opt/eclipse; chown -R hdfs:hadoop /opt/eclipse)
　　连接hadoop环境为：CDH 5.2 on a Single Linux Node in Pseudo-distributed Mode（安装参考附录）

　　5. Windows 7一样可使用这个hadoop-eclipse-plugin-2.5.0.jar
　　附录：
　　Installing CDH 5 with YARN on a Single Linux Node in Pseudo-distributed mode
　　分区考虑，不要使用LVM
　　root -- >50G
　　opt -- >50G
　　swap -- 2倍系统内存
　　RAM -- >8GB
　　1. disable selinux and iptables
　　service iptables stop
　　chkconfig iptables off; chkconfig ip6tables off
　　setenforce 0
　　sed -i 's,SELINUX=enforcing,SELINUX=disabled,g' /etc/selinux/config
　　2. disable ipv6 and kernel parameters tuning
　　echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf
　　echo "vm.swappiness = 0" >> /etc/sysctl.conf
　　echo 'net.ipv4.tcp_retries2 = 2' >> /etc/sysctl.conf
　　echo 'vm.overcommit_memory = 1' >> /etc/sysctl.conf
　　echo "fs.file-max = 6815744" >> /etc/sysctl.conf
　　echo "fs.aio-max-nr = 1048576" >> /etc/sysctl.conf
　　echo "net.core.rmem_default = 262144" >> /etc/sysctl.conf
　　echo "net.core.wmem_default = 262144" >> /etc/sysctl.conf
　　echo "net.core.rmem_max = 16777216" >> /etc/sysctl.conf
　　echo "net.core.wmem_max = 16777216" >> /etc/sysctl.conf
　　echo "net.ipv4.tcp_rmem = 4096 262144 16777216" >> /etc/sysctl.conf
　　echo "net.ipv4.tcp_wmem = 4096 262144 16777216" >> /etc/sysctl.conf
　　only on ResourceManager and JobHistory Server
　　echo "net.core.somaxconn = 1000" >> /etc/sysctl.conf
　　sysctl -p
　　echo "echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled" >> /etc/rc.local
　　echo "echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag" >> /etc/rc.local
　　echo "echo no > /sys/kernel/mm/redhat_transparent_hugepage/khugepaged/defrag" >> /etc/rc.local
　　3. vi /etc/hosts to add all hosts FQDN, like below:
　　192.168.1.10    test.gkytech.com test
　　vi /etc/sysconfig/network to set FQDN
　　cat  /etc/yum.repos.d/iso.repo
　　[iso]
　　name=iso
　　baseurl=http://192.168.1.10/centos65
　　enable=1
　　gpgcheck=0
　　EOF
　　yum -y install ntp openssh-clients lzo
　　service ntpd start; chkconfig ntpd on
　　vi /etc/security/limits.conf
　　*         soft    core         unlimited
　　*         hard    core         unlimited
　　*         soft    nofile       65536
　　*         hard    nofile       65536
　　*         soft    nproc        unlimited
　　*         hard    nproc        unlimited
　　*         soft    memlock      unlimited
　　*         hard    memlock      unlimited
　　vi /etc/grub.conf
　　add "elevator=deadline"(no quotes) at the end of kernel line
　　reboot to take effect
　　4. cat  /etc/yum.repos.d/cdh520.repo
　　[cm520]
　　name=cm520
　　baseurl=http://192.168.1.10/cdh520
　　enable=1
　　gpgcheck=0
　　EOF
　　rpm -ivh jdk-7u67-linux-x64.rpm
　　echo 'export JAVA_HOME=/usr/java/default' >> /etc/profile
　　echo 'export PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile
　　source /etc/profile
　　# To install Hadoop with YARN
　　yum -y install hadoop-conf-pseudo
　　# Step 1: Format the NameNode
　　sudo -u hdfs hdfs namenode -format
　　# Step 2: Start HDFS
　　for x in `cd /etc/init.d ; ls hadoop-hdfs-*` ; do service $x start ; done
　　# Step 3: Create the directories needed for Hadoop processes
　　/usr/lib/hadoop/libexec/init-hdfs.sh
　　# Step 4: Verify the HDFS File Structure
　　sudo -u hdfs hadoop fs -ls -R /
　　# Step 5: Start YARN
　　optional: to replace localhost with ip address, so you can access web ui
　　vi /etc/hadoop/conf/core-site.xml
　　vi /etc/hadoop/conf/mapred-site.xml
　　service hadoop-yarn-resourcemanager start
　　service hadoop-yarn-nodemanager start
　　service hadoop-mapreduce-historyserver start
　　# Step 6: Create User Directories
　　useradd test
　　sudo -u hdfs hadoop fs -mkdir /user/test
　　sudo -u hdfs hadoop fs -chown test /user/test
　　# Step 7: Testing
　　su - test
　　hadoop jar /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar pi 10 100