-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathspark-defaults-master.conf
More file actions
17 lines (17 loc) · 1.99 KB
/
Copy pathspark-defaults-master.conf
File metadata and controls
17 lines (17 loc) · 1.99 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
spark.eventLog.enabled=EVENT_LOG_ENABLED
spark.eventLog.dir=hdfs://localhost:8021/directory
spark.serializer=SPARK_SERIALIZER_CLASS
spark.driver.memory=SPARK_DRIVER_MEMORY
spark.executor.extraJavaOptions=SPARK_EXTRA_JVM_OPTIONS
#The maximum number of completed applications to display. Older applications will be dropped from the UI to maintain this limit.
spark.deploy.retainedApplications=MAX_RETAIN_APPLICATIONS
#The maximum number of completed drivers to display. Older drivers will be dropped from the UI to maintain this limit.
spark.deploy.retainedDrivers=MAX_RETAIN_DRIVERS
#Whether the standalone cluster manager should spread applications out across nodes or try to consolidate them onto as few nodes as possible. Spreading out is usually better for data locality in HDFS, but consolidating is more efficient for compute-intensive workloads.
spark.deploy.spreadOut=DEPLOY_SPREAD_OUT
#Default number of cores to give to applications in Spark's standalone mode if they don't set spark.cores.max. If not set, applications always get all available cores unless they configure spark.cores.max themselves. Set this lower on a shared cluster to prevent users from grabbing the whole cluster by default.
spark.deploy.defaultCores=DEPLOY_DEFAULT_CORES
#Limit on the maximum number of back-to-back executor failures that can occur before the standalone cluster manager removes a faulty application. An application will never be removed if it has any running executors. If an application experiences more than spark.deploy.maxExecutorRetries failures in a row, no executors successfully start running in between those failures, and the application has no running executors then the standalone cluster manager will remove the application and mark it as failed. To disable this automatic removal, set spark.deploy.maxExecutorRetries to -1.
spark.deploy.maxExecutorRetries=DEPLOY_MAX_RETRIES
#Number of seconds after which the standalone deploy master considers a worker lost if it receives no heartbeats.
spark.worker.timeout=WORKER_TIMEOUT