[大数据]Hadoop3 集群安装

安装虚拟机 (刚刚开始写，见谅)

首先在虚拟机中安装一台centos7，安装方式为最小安装，设置虚拟机网络为NET模式，配置相关的网络和主机名

将hadoop和java上传至centos7中

解压hadoop和java

配置java

配置hadoop

配置hadoop-env.sh

让hadoop知道jdk的路径

export JAVA_HOME=/opt/module/jdk

配置core-site.xml

 	<property>
        <name>fs.defaultFS</name>
        <value>hdfs://hadoop101:8020</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>file:///opt/module/hadoop/data</value>
    </property>
    <property>
        <name>hadoop.proxyuser.atguigu.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.atguigu.groups</name>
        <value>*</value>
    </property>
	<property>
        <name>hadoop.http.staticuser.user</name>
        <value>lee</value>
    </property>

配置hadfs-site.xml

    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:///opt/module/hadoop/data/namenode</value>
    </property>

    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:///opt/module/hadoop/data/datanode</value>
    </property>
    
    <property>
        <name>dfs.namenode.checkpoint.dir</name>
       	<value>file:///opt/module/hadoop/data/namesecondary</value>
    </property>

	<property>
        <name>dfs.client.datanode-restart.timeout</name>
        <value>30</value>
    </property>

    <property>
        <name>dfs.namenode.http-address</name>
        <value>hadoop101:50070</value>
    </property>


    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>hadoop103:50090</value>
    </property>

配置yarn-site.xml

 	<property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>hadoop102</value>
    </property>

    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>
	<!-- 开启日志聚集功能 -->
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>
    <!-- 设置日志聚集服务器地址 -->
    <property>
        <name>yarn.log.server.url</name>
        <value>http://hadoop102:19888/jobhistory/logs</value>
    </property>
    <!-- 设置日志保留时间为7天 -->
    <property>
        <name>yarn.log-aggregation.retain-seconds</name>
        <value>604800</value>
    </property>

配置MapReduce

	<property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>

    <!--history address-->
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>hadoop101:10020</value>
    </property>
    
    <!--history web address-->
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>hadoop101:19888</value>
    </property>