SparkConf设置的key / value 保存在SparkConf名为settings的变量中,该变量类型为ConcurrentHashMap
属性 | SparkConf提供的封装方法 |
---|---|
spark.master | setMaster |
spark.app.name | setAppName |
spark.default.parallelism | 无 |
val conf = new SparkConf() conf.setAppName("my-spark-01") conf.setMaster("local") conf.set("spark.default.parallelism", "3")
def setAppName(name: String): SparkConf = { set("spark.app.name", name) }
def set(key: String, value: String): SparkConf = { set(key, value, false) }
private val settings = new ConcurrentHashMap[String, String]() ... private[spark] def set(key: String, value: String, silent: Boolean): SparkConf = { ... settings.put(key, value) this }