hadoop HA 配置文件

65rew111

fs.defaultFS     hdfs://rongxinhadoop   
这里的 mycluster为HA集群的逻辑名，与hdfs-site.xml中的dfs.nameservices配置一致  hadoop.tmp.dir     /data/hadoop1/HAtmp3   
这里的路径默认是NameNode、DataNode、JournalNode等存放数据的公共目录。用户也可单独指定每类数据的存储目录。这里目录结构需要自己先创建好
ha.zookeeper.quorum   master:2181,slave1:2181,slave2:2181
这里是zk集群配置中各节点的地址和端口。  
注意：数量一定是奇数而且和zoo.cfg中配置的一致
--------------------------------------------------------------------------------------------------
dfs.replication     2     配置副本数量  
dfs.namenode.name.dir     file:/data/hadoop1/HAname3     namenode元数据存储目录  dfs.datanode.data.dir     file:/data/hadoop1/HAdata3     datanode数据存储目录  dfs.nameservices          rongxinhadoop         指定HA命名服务，可随意起名，   core-site.xml中fs.defaultFS配置需要引用它  
dfs.ha.namenodes.rongxinhadoop         nn1,nn2         指定集群下NameNode逻辑名  
dfs.namenode.rpc-address.rongxinhadoop.nn1           master:9000  
dfs.namenode.rpc-address.rongxinhadoop.nn2           slave1:9000  
dfs.namenode.http-address.rongxinhadoop.nn1           master:50070  
dfs.namenode.http-address.rongxinhadoop.nn2           slave1:50070  
dfs.namenode.servicerpc-address.rongxinhadoop.nn1            master:53310   dfs.namenode.servicerpc-address.rongxinhadoop.nn2            slave1:53310  
dfs.ha.automatic-failover.enabled.rongxinhadoop          true         故障失败是否自动切换   dfs.namenode.shared.edits.dir         qjournal://master:8485;slave1:8485;slave2:8485/rongxinhadoop         配置JournalNode，包含三部分：       
1.qjournal 前缀表名协议；       
2.然后就是三台部署JournalNode的主机host/ip:端口，三台机器之间用分号分隔；       
3.最后的hadoop-journal是journalnode的命名空间，可以随意取名。            dfs.journalnode.edits.dir         /data/hadoop1/HAjournal3/         journalnode的本地数据存放目录 dfs.client.failover.proxy.provider.rongxinhadoop   
org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider   指定mycluster出故障时执行故障切换的类        
dfs.ha.fencing.methods        sshfence   ssh的操作方式执行故障切换      
dfs.ha.fencing.ssh.private-key-files        /home/hadoop1/.ssh/id_rsa   如果使用ssh进行故障切换，使用ssh通信时用的密钥存储的位置   
dfs.ha.fencing.ssh.connect-timeout    1000      dfs.namenode.handler.count    10  
--------------------------------------------------------------------------------------------------
mapreduce.framework.name yarn   mapreduce.jobhistory.address master:10020  mapreduce.jobhistory.webapp.address master:19888  mapreduce.jobhistory.intermediate-done-dir /data/hadoop1/mr_history/HAtmp3 Directory where history files are written by MapReduce jobs.  mapreduce.jobhistory.done-dir /data/hadoop1/mr_history/HAdone3 Directory where history files are managed by the MR JobHistory Server.
--------------------------------------------------------------------------------------------------
<?xml version="1.0"?>
<!-- Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. See accompanying LICENSE file. -->
-<configuration>
<!-- Site specific YARN configuration properties -->

-<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>

-<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>clusterrm</value>
</property>

-<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>

-<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>master</value>
</property>

-<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>slave1</value>
</property>

-<property>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>

-<property>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>

-<property>
<name>yarn.resourcemanager.zk-address</name>
<value>master:2181,slave1:2181,slave2:2181</value>
</property>

-<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>

-<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<!-- set the proxy server -->

<!-- set history server -->

-<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<!-- set the timeline server -->

-<property>
<description>The hostname of the Timeline service web application.</description>
<name>yarn.timeline-service.hostname</name>
<value>master</value>
</property>

-<property>
<description>Address for the Timeline server to start the RPC server.</description>
<name>yarn.timeline-service.address</name>
<value>master:10200</value>
</property>

-<property>
<description>The http address of the Timeline service web application.</description>
<name>yarn.timeline-service.webapp.address</name>
<value>master:8188</value>
</property>

-<property>
<description>The https address of the Timeline service web application.</description>
<name>yarn.timeline-service.webapp.https.address</name>
<value>master:8190</value>
</property>

-<property>
<description>Handler thread count to serve the client RPC requests.</description>
<name>yarn.timeline-service.handler-thread-count</name>
<value>10</value>
</property>

-<property>
<description>Enables cross-origin support (CORS) for web services where cross-origin web response headers are needed. For example, javascript making a web services request to the timeline server.</description>
<name>yarn.timeline-service.http-cross-origin.enabled</name>
<value>false</value>
</property>

-<property>
<description>Comma separated list of origins that are allowed for web services needing cross-origin (CORS) support. Wildcards (*) and patterns allowed</description>
<name>yarn.timeline-service.http-cross-origin.allowed-origins</name>
<value>*</value>
</property>

-<property>
<description>Comma separated list of methods that are allowed for web services needing cross-origin (CORS) support.</description>
<name>yarn.timeline-service.http-cross-origin.allowed-methods</name>
<value>GET,POST,HEAD</value>
</property>

-<property>
<description>Comma separated list of headers that are allowed for web services needing cross-origin (CORS) support.</description>
<name>yarn.timeline-service.http-cross-origin.allowed-headers</name>
<value>X-Requested-With,Content-Type,Accept,Origin</value>
</property>

-<property>
<description>The number of seconds a pre-flighted request can be cached for web services needing cross-origin (CORS) support.</description>
<name>yarn.timeline-service.http-cross-origin.max-age</name>
<value>1800</value>
</property>

-<property>
<description>Indicate to clients whether Timeline service is enabled or not. If enabled, the TimelineClient library used by end-users will post entities and events to the Timeline server.</description>
<name>yarn.timeline-service.enabled</name>
<value>true</value>
</property>

-<property>
<description>Store class name for timeline store.</description>
<name>yarn.timeline-service.store-class</name>
<value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
</property>

-<property>
<description>Enable age off of timeline store data.</description>
<name>yarn.timeline-service.ttl-enable</name>
<value>true</value>
</property>

-<property>
<description>Time to live for timeline store data in milliseconds.</description>
<name>yarn.timeline-service.ttl-ms</name>
<value>604800000</value>
</property>
</configuration>