[大数据]hadoop常用配置文件

目录

Hadoop2.x配置：

1.hadoop-env.sh/yarn-env.sh/mapred-env.sh

2.core-site.xml

3.hdfs-site.xml

4.yarn-site.xml

5.mapred-site.xml

6.xsync

7.vim /etc/profile

8.myhadoop.sh

9.jpsall

Hadoop3.x配置：

1.core-site.xml

2.yarn-site.xml

3.mapred-site.xml

4.hdfs-site.xml

5.xsync

6.myhadoop.sh

7.jpsall

8. vim /etc/profile.d/my_env.sh

Hadoop2.x配置：

1.hadoop-env.sh/yarn-env.sh/mapred-env.sh

export JAVA_HOME=/opt/module/jdk1.8.0_144

2.core-site.xml

<!-- 指定HDFS中NameNode的地址 -->

<property>

<name>fs.defaultFS</name>

????<value>hdfs://hadoop15:9000</value>

</property>

<!-- 指定Hadoop运行时产生文件的存储目录 -->

<property>

????????<name>hadoop.tmp.dir</name>

????????<value>/opt/module/hadoop-2.7.2/data/tmp</value>

</property>

3.hdfs-site.xml

<!-- 指定HDFS副本的数量 -->

<property>

????????<name>dfs.replication</name>

????????<value>3</value>

</property>

<!-- 指定Hadoop辅助名称节点主机配置 -->

<property>

??????<name>dfs.namenode.secondary.http-address</name>

??????<value>hadoop17:50090</value>

</property>

4.yarn-site.xml

<!-- Reducer获取数据的方式 -->

<property>

????????<name>yarn.nodemanager.aux-services</name>

????????<value>mapreduce_shuffle</value>

</property>

<!-- 指定YARN的ResourceManager的地址 -->

<property>

????????<name>yarn.resourcemanager.hostname</name>

????????<value>hadoop16</value>

</property>

<!-- 日志聚集功能使能 -->

<property>

????????<name>yarn.log-aggregation-enable</name>

????????<value>true</value>

</property>

<!-- 日志保留时间设置7天 -->

<property>

????????<name>yarn.log-aggregation.retain-seconds</name>

????????<value>604800</value>

</property>

5.mapred-site.xml

<!-- 指定MR运行在YARN上 -->

<property>

????????<name>mapreduce.framework.name</name>

????????<value>yarn</value>

</property>

<!-- 历史服务器端地址 -->

<property>

????????<name>mapreduce.jobhistory.address</name>

????????<value>hadoop15:10020</value>

</property>

<!-- 历史服务器web端地址 -->

<property>

????????<name>mapreduce.jobhistory.webapp.address</name>

????????<value>hadoop15:19888</value>

</property>

6.xsync

#!/bin/bash

#1 获取输入参数个数，如果没有参数，直接退出

pcount=$#

if((pcount==0)); then

echo no args;

exit;

fi

#2 获取文件名称

p1=$1

fname=`basename $p1`

echo fname=$fname

#3 获取上级目录到绝对路径

pdir=`cd -P $(dirname $p1); pwd`

echo pdir=$pdir

#4 获取当前用户名称

user=`whoami`

#5 循环

for((host=16; host<18; host++)); do

????????echo ------------------- hadoop$host --------------

????????rsync -rvl $pdir/$fname $user@hadoop$host:$pdir

done

7.vim /etc/profile

#JAVA_HOME

export JAVA_HOME=/opt/module/jdk1.8.0_144

export PATH=$PATH:$JAVA_HOME/bin

#HADOOP_HOME

export HADOOP_HOME=/opt/module/hadoop-2.7.2

export PATH=$PATH:$HADOOP_HOME/bin

export PATH=$PATH:$HADOOP_HOME/sbin

8.myhadoop.sh

#!/bin/bash

if [ $# -lt 1 ]

then

????????echo "No Args Input..."

????????exit ;

fi

case $1 in

"start")

????????echo " =================== 启动 hadoop 集群 ==================="

????????echo " --------------- 启动 hdfs ---------------"

????????ssh hadoop15 "/opt/module/hadoop-2.7.2/sbin/start-dfs.sh"

????????echo " --------------- 启动 yarn ---------------"

????????ssh hadoop16 "/opt/module/hadoop-2.7.2/sbin/start-yarn.sh"

????????echo " --------------- 启动 historyserver ---------------"

????????ssh hadoop15 "/opt/module/hadoop-2.7.2/sbin/mr-jobhistory-daemon.sh?start historyserver"

;;

"stop")

????????echo " =================== 关闭 hadoop 集群 ==================="

????????echo " --------------- 关闭 historyserver ---------------"

????????ssh hadoop15 "/opt/module/hadoop-2.7.2/sbin/mr-jobhistory-daemon.sh?start historyserver"

????????echo " --------------- 关闭 yarn ---------------"

????????ssh hadoop16 "/opt/module/hadoop-2.7.2/sbin/stop-yarn.sh" echo " --------------- 关闭 hdfs ---------------"

????????ssh hadoop15 "/opt/module/hadoop-2.7.2/sbin/stop-dfs.sh"

;;

*)

????????echo "Input Args Error..."

;;

esac

9.jpsall

#!/bin/bash

for host in hadoop15 hadoop16 hadoop17

do

????????echo =============== $host ===============

????????ssh $host /opt/module/jdkjdk1.8.0_144/bin/jps

done

Hadoop3.x配置：

1.core-site.xml

<!--指定NameNode的地址-->

<property>

????????<name>fs.defaultFS</name>

????????<value>hdfs://hadoop25:8020</value>

</property>

<!--指定hadoop数据的存储目录-->

<property>

????????<name>hadoop.tmp.dir</name>

????????<value>/opt/module/hadoop-3.1.3/data</value>

</property>

<!--配置HDFS网页登录使用的静态用户为atguigu -->

<property>

????????<name>hadoop.http.staticuser.user</name>

????????<value>real</value>

</property>

2.yarn-site.xml

<!-- 指定 MR 走 shuffle -->

<property>

????????<name>yarn.nodemanager.aux-services</name>

????????<value>mapreduce_shuffle</value>

</property>

<!-- 指定 ResourceManager 的地址-->

<property>

????????<name>yarn.resourcemanager.hostname</name>

????????<value>hadoop26</value>

</property>

<!-- 环境变量的继承 -->

<property>

????????<name>yarn.nodemanager.env-whitelist</name>

????????<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CO NF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>

</property>

<!--开启日志聚集功能-->

<property>

????????<name>yarn.log-aggregation-enable</name>

????????<value>true</value>

</property>

<!--设置日志聚集服务器地址-->

<property>

????????<name>yarn.log.server.url</name>

????????<value>http://hadoop25:19888/jobhistory/logs</value>

</property>

<!--设置日志保留时间为7天-->

<property>

????????<name>yarn.log-aggregation.retain-seconds</name>

????????<value>604800</value>

</property>

3.mapred-site.xml

<!-- 指定 MapReduce 程序运行在 Yarn 上 -->

<property>

????????<name>mapreduce.framework.name</name>

????????<value>yarn</value>

</property>

<!--历史服务器端地址-->

<property>

????????<name>mapreduce.jobhistory.address</name>

????????<value>hadoop25:10020</value>

</property>

<!--历史服务器web端地址-->

<property>

????????<name>mapreduce.jobhistory.webapp.address</name>

????????<value>hadoop25:19888</value>

</property>

4.hdfs-site.xml

<!--nn web端访问地址-->

<property>

????????<name>dfs.namenode.http-address</name>

????????<value>hadoop25:9870</value>

</property>

<!--2nn web端访问地址-->

<property>

????????<name>dfs.namenode.secondary.http-address</name>

????????<value>hadoop27:9868</value>

</property>

5.xsync

#!/bin/bash

#1. 判断参数个数

if [ $# -lt 1 ]

then

????????echo Not Enough Arguement!

????????exit;

fi

#2. 遍历集群所有机器

for host in hadoop25 hadoop26 hadoop27

do

????????echo ==================== ?$host ?====================

????????#3. 遍历所有目录，挨个发送

????????for file in $@

????????do

????????????????#4. 判断文件是否存在

????????????????if [ -e $file ]

????????????????????????then

????????????????????????????????#5. 获取父目录

????????????????????????????????pdir=$(cd -P $(dirname $file); pwd)

????????????????????????????????#6. 获取当前文件的名称

????????????????????????????????fname=$(basename $file)

????????????????????????????????ssh $host "mkdir -p $pdir"

????????????????????????????????rsync -av $pdir/$fname $host:$pdir

????????????????????????else

????????????????????????????????echo $file does not exists!

????????????????fi

????????done

done

6.myhadoop.sh

#!/bin/bash

if [ $# -lt 1 ]

then

????????echo "No Args Input..."

????????exit ;

fi

case $1 in

"start")

????????echo " =================== 启动 hadoop 集群 ==================="

????????echo " --------------- 启动 hdfs ---------------"

????????ssh hadoop25 "/opt/module/hadoop-3.1.3/sbin/start-dfs.sh"

????????echo " --------------- 启动 yarn ---------------"

????????ssh hadoop26 "/opt/module/hadoop-3.1.3/sbin/start-yarn.sh"

????????echo " --------------- 启动 historyserver ---------------"

????????ssh hadoop25 "/opt/module/hadoop-3.1.3/bin/mapred --daemon start historyserver"

;;

"stop")

????????echo " =================== 关闭 hadoop 集群 ==================="

????????echo " --------------- 关闭 historyserver ---------------"

????????ssh hadoop25 "/opt/module/hadoop-3.1.3/bin/mapred --daemon stop historyserver"

????????echo " --------------- 关闭 yarn ---------------"

????????ssh hadoop26 "/opt/module/hadoop-3.1.3/sbin/stop-yarn.sh" echo " --------------- 关闭 hdfs ---------------"

????????ssh hadoop25 "/opt/module/hadoop-3.1.3/sbin/stop-dfs.sh"

;;

*)

????????echo "Input Args Error..."

;;

esac

7.jpsall

#!/bin/bash

for host in hadoop25 hadoop26 hadoop27

do

????????echo =============== $host ===============

????????ssh $host jps

done

8. vim /etc/profile.d/my_env.sh

#JAVA_HOME

export JAVA_HOME=/opt/module/jdk1.8.0_212

export PATH=$PATH:$JAVA_HOME/bin

#HADOOP_HOME

export HADOOP_HOME=/opt/module/hadoop-3.1.3

export PATH=$PATH:$HADOOP_HOME/bin

export PATH=$PATH:$HADOOP_HOME/sbin

?