[大数据]hive自动化上传文件到表

自动化实现上传文件到表

#!/bin/bash
. /etc/profile
. ~/.bash_profile
. ~/.bashrc


error() {
       if [[ $? != 0 ]]; then
       echo "$@  运行失败"
       exit 1
       fi
}

# 待上传表名
inset_table_name=$1
# 待上传文件名
file_name=$2
# 分区名
pt=$3
load_path="/tmp/load_path/"

curr_date=`date +%F|sed 's/-//g'`

eg_sql="
create table  tmp.xx_xxx_${curr_date}
(
filed_list string 
,label_one string 
)partitioned by (pt string comment 'YYMMDD数据入库时间')
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS textfile
"

if [[ $1 = "help" ]];then 
   echo -e "\033[40;31m   追加写入分区表 :入参配置 4个参数：待上传表名+待上传文件名+分区名 + 1 [第四个参数为1，表示追加]  \033[0m"
   echo "ds_load upload_table file_name pt 1"
   echo -e "\033[40;31m   覆盖写入分区表 :入参配置 3个参数：待上传表名+待上传文件+分区名【不能为数字1】 \033[0m"
   echo "ds_load upload_table file_name pt"
   echo -e "\033[40;31m   追加写入非分区表 :入参配置 3个参数：待上传表名+待上传文件 + 1【数字1，表示追加】 \033[0m"
   echo "ds_load upload_table file_name 1"
   echo -e "\033[40;31m   覆盖写入非分区表 :入参配置 2个参数：待上传表名+待上传文件  \033[0m"
   echo "ds_load upload_table file_name"
   echo "建表语句样例：${eg_sql}"
   exit 0
fi 

if [[ $# < 2 ]];then 
  echo "参数个数有问题，请使用ds_load help 查看用法"
  exit 0
fi



hadoop fs -put ${file_name} ${load_path}${file_name}
error "上传失败，请检查HDFS目录,删除失败文件"

load_sql_pt_into="load data inpath '${load_path}${file_name}'  into table ${inset_table_name} partition(pt='${pt}')"

load_sql_pt="load data inpath '${load_path}${file_name}' overwrite  into table ${inset_table_name} partition(pt='${pt}')"

load_sql_into="load data inpath '${load_path}${file_name}'   into table ${inset_table_name} "

load_sql_ow="load data inpath '${load_path}${file_name}' overwrite  into table ${inset_table_name} "


#  追加写入分区表
if [[ $# = 4  ]] && [[ $4 = 1 ]];then 
  sh ~/bee_u_e.sh "${load_sql_pt_into}"
  error  "上传"
  echo "上传成功"
# 覆盖写入分区表，分区名不能为1
elif [[  $# = 3 ]] && [[ $3 != 1 ]];then
  sh ~/bee_u_e.sh "${load_sql_pt}"
  error  "上传"
  echo "上传成功"
# 追加写入非分区表
elif [[  $# = 3 ]] && [[ $3 = 1 ]];then
  sh ~/bee_u_e.sh "${load_sql_into}"
  error  "上传"
  echo "上传成功"
# 覆盖写入分区表
elif [[ $# = 2 ]];then 
  sh ~/bee_u_e.sh "${load_sql_ow}"
  error  "上传"
  echo "上传成功"
else 
  echo "请检查参数个数"
  echo -e "\033[40;31m 使用ds_load help 查看用法 \033[0m"
  exit 0
fi