[大数据]spark 优化

背景代码

System.setProperty("HADOOP_USER_NAME", "root")
System.setProperty("hadoop.home.dir", "C:\\winutils")
val conf: SparkConf = new SparkConf().setAppName("dws_member_import").setMaster("local[4]")
val sparkSession: SparkSession = SparkSession.builder().config("dfs.client.use.datanode.hostname", "true").config(conf).enableHiveSupport().getOrCreate()
val ssc: SparkContext = sparkSession.sparkContext
ssc.hadoopConfiguration.set("fs.defaultFS", "hdfs://mycluster")
ssc.hadoopConfiguration.set("dfs.nameservices", "mycluster")

import sparkSession.implicits._
val 

了解本专栏 订阅专栏 解锁全文