Hadoop集群版 高可用
所需脚本网址
jqop https://blog.csdn.net/qq_51903852/article/details/121120429
xsync https://blog.csdn.net/qq_51903852/article/details/121141166
zkop https://blog.csdn.net/qq_51903852/article/details/121120516
[root@mihaoyu152 install]# tar -zxvf hadoop-2.6.0-cdh5.14.2.tar.gz -C ../soft[root@mihaoyu152 soft]# mv hadoop-2.6.0-cdh5.14.2/ hadoop260[root@mihaoyu152 hadoop]# vi ./hadoop-env.sh 25 export JAVA_HOME=/opt/soft/jdk180[root@mihaoyu152 hadoop]# vi ./yarn-env.sh 23 export JAVA_HOME=/opt/soft/jdk180[root@mihaoyu152 hadoop]# vi ./mapred-env.sh16 export JAVA_HOME=/opt/soft/jdk180[root@mihaoyu152 hadoop]# vi ./core-site.xml <configuration> <!-- 指定的hdfs的nameservice 为mycluster --> <property><name>fs.defaultFS</name><value>hdfs://mycluster/</value> </property> <!-- 指定的hadoop工作目录 --> <property><name>hadoop.tmp.dir</name><value>/opt/soft/hadoop260/hadooptmp/</value> </property> <!-- 指定zookeeper集群访问地址 --> <property><name>ha.zookeeper.quorum</name><value>mihaoyu153:2181,mihaoyu154:2181,mihaoyu155:2181</value> </property> <!-- 配置为了解决以后其他组件连接HDFS集群 --> <property><name>hadoop.proxyuser.bigdata.hosts</name><value>*</value> </property> <property><name>hadoop.proxyuser.bigdata.groups</name><value>*</value> </property></configuration>[root@mihaoyu152 hadoop]# vi ./hdfs-site.xml <configuration> <!-- 指定副本数 --> <property><name>dfs.replication</name><value>3</value> </property> <!-- 指定HDFS的nameservices为mycluster，需要跟core-site.xml中保持一致 --> <property><name>dfs.nameservices</name><value>mycluster</value> </property> <!-- 设置mycluster集群有两个namenode，分别为nn1，nn2 --> <property><name>dfs.ha.namenodes.mycluster</name><value>nn1,nn2</value> </property> <!-- 配置nn1的RPC通信地址 --> <property><name>dfs.namenode.rpc-address.mycluster.nn1</name><value>mihaoyu152:9000</value> </property> <!-- 配置nn1的http通信地址 --> <property><name>dfs.namenode.http-address.mycluster.nn1</name><value>mihaoyu152:50070</value> </property> <!-- 配置nn2的RPC通信地址 --> <property><name>dfs.namenode.rpc-address.mycluster.nn2</name><value>mihaoyu153:9000</value> </property> <!-- 配置nn2的http通信地址 --> <property><name>dfs.namenode.http-address.mycluster.nn2</name><value>mihaoyu153:50070</value> </property> <!-- 指定JournalNode 在本地磁盘存放数据的位置 --> <property><name>dfs.journalnode.edits.dir</name><value>/opt/soft/hadoop260/journaldata</value> </property> <!-- 指定NameNode的edits元数据在journalNode上的服务器 --> <property><name>dfs.namenode.shared.edits.dir</name><value>qjournal://mihaoyu152:8485;mihaoyu153:8485;mihaoyu154:8485/mycluster</value> </property> <!-- 开启NameNode 自动切换 --> <property><name>dfs.ha.automatic-failover.enabled</name><value>true</value> </property> <!-- 配置namenode失败自动切换的实现方式 --> <property><name>dfs.client.failover.proxy.provider.mycluster</name><value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value> </property> <!-- 配置隔离机制方法 --> <property><name>dfs.ha.fencing.methods</name><value> sshfence shell(/bin/true)</value> </property> <!-- 使用sshfence隔离机制时需要ssh免密登陆 --> <property><name>dfs.ha.fencing.ssh.private-key-files</name><value>/root/.ssh/id_rsa</value> </property> <!-- 配置sshfence隔离机制超时时间 --> <property><name>dfs.ha.fencing.ssh.connect-timeout</name><value>30000</value> </property> <property><name>dfs.webhdfs.enabled</name><value>true</value> </property></configuration>[root@mihaoyu152 hadoop]# cp mapred-site.xml.template mapred-site.xml[root@mihaoyu152 hadoop]# vi ./mapred-site.xml<configuration> <!-- 指定mapreduce运算时资源调度框架为 yarn 模式 --> <property><name>mapreduce.framework.name</name><value>yarn</value> </property> <!-- 配置mapreduce历史服务器地址 端口号 --> <property><name>mapreduce.jobhistory.address</name><value>mihaoyu155:10020</value> </property> <!-- 配置mapreduce历史服务器WEB访问地址 --> <property><name>mapreduce.jobhistory.webapp.address</name><value>mihaoyu155:19888</value> </property></configuration>[root@mihaoyu152 hadoop]# vi ./yarn-site.xml<configuration> <!-- 开启高可用 --> <property><name>yarn.resourcemanager.ha.enabled</name><value>true</value> </property> <!-- 指定ResourceManager的标识：yrc --> <property><name>yarn.resourcemanager.cluster-id</name><value>yrc</value> </property> <!-- 指定RM的名字 --> <property><name>yarn.resourcemanager.ha.rm-ids</name><value>rm1,rm2</value> </property> <!-- 指定rm1服务器 --> <property><name>yarn.resourcemanager.hostname.rm1</name><value>mihaoyu152</value> </property> <!-- 指定rm2服务器 --> <property><name>yarn.resourcemanager.hostname.rm2</name><value>mihaoyu153</value> </property> <!-- 指定rm 被管理的zk 地址 --> <property><name>yarn.resourcemanager.zk-address</name><value>mihaoyu153:2181,mihaoyu154:2181,mihaoyu155:2181</value> </property> <!-- 运行mapreduce任务时需要使用的服务 --> <property><name>yarn.nodemanager.aux-services</name><value>mapreduce_shuffle</value> </property> <!-- 开启yarn集群的日志聚合功能 --> <property><name>yarn.log-aggregation-enable</name><value>true</value> </property> <!-- 设置日志保存时间 --> <property><name>yarn.log-aggregation.retain-seconds</name><value>86400</value> </property> <!-- 启动rm自动恢复功能 --> <property><name>yarn.resourcemanager.recovery.enabled</name><value>true</value> </property> <!-- 制定rm 状态信息存储在zookeeper集群上 --> <property><name>yarn.resourcemanager.store.class</name><value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value> </property></configuration>[root@mihaoyu152 hadoop]# vi ./slavesmihaoyu152 mihaoyu153 mihaoyu154 mihaoyu155[root@mihaoyu152 hadoop]# vi /etc/profile#hadoopexport HADOOP_HOME=/opt/soft/hadoop260export HADOOP_MAPRED_HOME=$HADOOP_HOMEexport HADOOP_COMMON_HOME=$HADOOP_HOMEexport HADOOP_HDFS_HOME=$HADOOP_HOMEexport YARN_HOME=$HADOOP_HOMEexport HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/nativeexport HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin[root@mihaoyu152 soft]# source /etc/profile[root@mihaoyu152 soft]# xsync hadoop260[root@mihaoyu152 soft]# xsync /etc/profile
第一次启动集群服务
1、先将zookeeper启动[root@mihaoyu152 soft]# zkop start--------------- mihaoyu153 zookeeper start --------------- JMX enabled by default Using config: /opt/soft/zookeeper345/bin/../conf/zoo.cfg Starting zookeeper ... STARTED --------------- mihaoyu154 zookeeper start --------------- JMX enabled by default Using config: /opt/soft/zookeeper345/bin/../conf/zoo.cfg Starting zookeeper ... STARTED --------------- mihaoyu155 zookeeper start --------------- JMX enabled by default Using config: /opt/soft/zookeeper345/bin/../conf/zoo.cfg Starting zookeeper ... STARTED[root@mihaoyu152 soft]# zkop status--------------- mihaoyu153 zookeeper status --------------- JMX enabled by default Using config: /opt/soft/zookeeper345/bin/../conf/zoo.cfg Mode: follower --------------- mihaoyu154 zookeeper status --------------- JMX enabled by default Using config: /opt/soft/zookeeper345/bin/../conf/zoo.cfg Mode: leader --------------- mihaoyu155 zookeeper status --------------- JMX enabled by default Using config: /opt/soft/zookeeper345/bin/../conf/zoo.cfg Mode: follower2、启动journalnode[root@mihaoyu152 soft]# hadoop-daemon.sh start journalnodestarting journalnode, logging to /opt/soft/hadoop260/logs/hadoop-root-journalnode-mihaoyu152.out[root@mihaoyu152 soft]# ssh mihaoyu153 "source /etc/profile; hadoop-daemon.sh start journalnode"starting journalnode, logging to /opt/soft/hadoop260/logs/hadoop-root-journalnode-mihaoyu153.out[root@mihaoyu152 soft]# ssh mihaoyu154 "source /etc/profile; hadoop-daemon.sh start journalnode"starting journalnode, logging to /opt/soft/hadoop260/logs/hadoop-root-journalnode-mihaoyu154.out[root@mihaoyu152 soft]# jqop jps--------------- mihaoyu152 指令信息 --------------- jps5986 Jps5894 JournalNode --------------- mihaoyu153 指令信息 --------------- jps4676 QuorumPeerMain4807 Jps4748 JournalNode --------------- mihaoyu154 指令信息 --------------- jps3570 JournalNode3493 QuorumPeerMain3629 Jps --------------- mihaoyu155 指令信息 --------------- jps3378 Jps3318 QuorumPeerMain3、格式化 namenode mihaoyu152 mihaoyu153[root@mihaoyu152 soft]# hadoop namenode -format[root@mihaoyu152 soft]# cd ./hadoop260/将mihaoyu152格式化后的hadooptmp文件同步到mihaoyu153[root@mihaoyu152 hadoop260]# scp -r ./hadooptmp/ root@mihaoyu153:/opt/soft/hadoop260/VERSION 100% 205 235.0KB/s 00:00 seen_txid 100% 2 2.4KB/s 00:00 fsimage_0000000000000000000.md5 100% 62 106.3KB/s 00:00 fsimage_0000000000000000000 100% 320 565.5KB/s 00:00 4、初始化zookeeper[root@mihaoyu152 hadoop260]# hdfs zkfc -formatZK[root@mihaoyu153 hadoop260]# zkCli.sh 5、启动HDFS[root@mihaoyu152 hadoop260]# start-dfs.sh 21/11/04 21:40:35 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Starting namenodes on [mihaoyu152 mihaoyu153]mihaoyu153: starting namenode, logging to /opt/soft/hadoop260/logs/hadoop-root-namenode-mihaoyu153.out mihaoyu152: starting namenode, logging to /opt/soft/hadoop260/logs/hadoop-root-namenode-mihaoyu152.out mihaoyu153: starting datanode, logging to /opt/soft/hadoop260/logs/hadoop-root-datanode-mihaoyu153.out mihaoyu155: starting datanode, logging to /opt/soft/hadoop260/logs/hadoop-root-datanode-mihaoyu155.out mihaoyu154: starting datanode, logging to /opt/soft/hadoop260/logs/hadoop-root-datanode-mihaoyu154.out mihaoyu152: starting datanode, logging to /opt/soft/hadoop260/logs/hadoop-root-datanode-mihaoyu152.out Starting journal nodes [mihaoyu152 mihaoyu153 mihaoyu154]mihaoyu153: journalnode running as process 4748. Stop it first. mihaoyu154: journalnode running as process 3570. Stop it first. mihaoyu152: journalnode running as process 5894. Stop it first.21/11/04 21:40:47 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Starting ZK Failover Controllers on NN hosts [mihaoyu152 mihaoyu153]mihaoyu153: starting zkfc, logging to /opt/soft/hadoop260/logs/hadoop-root-zkfc-mihaoyu153.out mihaoyu152: starting zkfc, logging to /opt/soft/hadoop260/logs/hadoop-root-zkfc-mihaoyu152.out[root@mihaoyu152 hadoop260]# jqop jps--------------- mihaoyu152 指令信息 --------------- jps6581 DFSZKFailoverController5894 JournalNode6312 DataNode6185 NameNode6666 Jps --------------- mihaoyu153 指令信息 --------------- jps4883 NameNode4676 QuorumPeerMain5147 Jps4748 JournalNode4972 DataNode5085 DFSZKFailoverController --------------- mihaoyu154 指令信息 --------------- jps3570 JournalNode3493 QuorumPeerMain3767 Jps3659 DataNode --------------- mihaoyu155 指令信息 --------------- jps3318 QuorumPeerMain3495 Jps3407 DataNode6、启动yarn[root@mihaoyu152 hadoop260]# start-yarn.shstarting yarn daemons starting resourcemanager, logging to /opt/soft/hadoop260/logs/yarn-root-resourcemanager-mihaoyu152.out mihaoyu154: starting nodemanager, logging to /opt/soft/hadoop260/logs/yarn-root-nodemanager-mihaoyu154.out mihaoyu153: starting nodemanager, logging to /opt/soft/hadoop260/logs/yarn-root-nodemanager-mihaoyu153.out mihaoyu155: starting nodemanager, logging to /opt/soft/hadoop260/logs/yarn-root-nodemanager-mihaoyu155.out mihaoyu152: starting nodemanager, logging to /opt/soft/hadoop260/logs/yarn-root-nodemanager-mihaoyu152.out[root@mihaoyu152 hadoop260]# jqop jps--------------- mihaoyu152 指令信息 --------------- jps7139 Jps6581 DFSZKFailoverController6821 NodeManager5894 JournalNode6312 DataNode6185 NameNode6719 ResourceManager --------------- mihaoyu153 指令信息 --------------- jps4883 NameNode4676 QuorumPeerMain5188 NodeManager4748 JournalNode4972 DataNode5085 DFSZKFailoverController5310 Jps --------------- mihaoyu154 指令信息 --------------- jps3570 JournalNode3922 Jps3493 QuorumPeerMain3800 NodeManager3659 DataNode --------------- mihaoyu155 指令信息 --------------- jps3650 Jps3318 QuorumPeerMain3528 NodeManager3407 DataNode[root@mihaoyu152 hadoop260]# ssh mihaoyu153 "source /etc/profile; yarn-daemon.sh start resourcemanager"starting resourcemanager, logging to /opt/soft/hadoop260/logs/yarn-root-resourcemanager-mihaoyu153.out[root@mihaoyu152 hadoop260]# jqop jps--------------- mihaoyu152 指令信息 --------------- jps6581 DFSZKFailoverController6821 NodeManager5894 JournalNode6312 DataNode6185 NameNode7179 Jps6719 ResourceManager --------------- mihaoyu153 指令信息 --------------- jps4883 NameNode4676 QuorumPeerMain5188 NodeManager5352 ResourceManager5418 Jps4748 JournalNode4972 DataNode5085 DFSZKFailoverController --------------- mihaoyu154 指令信息 --------------- jps3570 JournalNode3493 QuorumPeerMain3800 NodeManager3659 DataNode3948 Jps --------------- mihaoyu155 指令信息 --------------- jps3318 QuorumPeerMain3528 NodeManager3676 Jps3407 DataNode7、启动historyserver[root@mihaoyu152 hadoop260]# ssh mihaoyu155 "source /etc/profile; mr-jobhistory-daemon.sh start historyserver"starting historyserver, logging to /opt/soft/hadoop260/logs/mapred-root-historyserver-mihaoyu155.out[root@mihaoyu152 hadoop260]# jqop jps--------------- mihaoyu152 指令信息 --------------- jps7219 Jps6581 DFSZKFailoverController6821 NodeManager5894 JournalNode6312 DataNode6185 NameNode6719 ResourceManager --------------- mihaoyu153 指令信息 --------------- jps5458 Jps4883 NameNode4676 QuorumPeerMain5188 NodeManager5352 ResourceManager4748 JournalNode4972 DataNode5085 DFSZKFailoverController --------------- mihaoyu154 指令信息 --------------- jps3570 JournalNode3972 Jps3493 QuorumPeerMain3800 NodeManager3659 DataNode --------------- mihaoyu155 指令信息 --------------- jps3797 Jps3318 QuorumPeerMain3528 NodeManager3720 JobHistoryServer3407 DataNode
目录 返回
首页