spark-user-configurable-options.template 3.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124
  1. // Application properties
  2. spark.app.name
  3. spark.driver.cores
  4. spark.driver.maxResultSize
  5. spark.driver.memory
  6. spark.executor.memory
  7. spark.extraListeners
  8. // spark.local.dir
  9. spark.logConf
  10. // spark.master
  11. // Runtime Environment
  12. spark.driver.extraClassPath
  13. // spark.driver.extraJavaOptions
  14. spark.driver.extraLibraryPath
  15. spark.driver.userClassPathFirst
  16. spark.executor.extraClassPath
  17. // spark.executor.extraJavaOptions
  18. spark.executor.extraLibraryPath
  19. spark.executor.instances
  20. spark.executor.logs.rolling.maxRetainedFiles
  21. spark.executor.logs.rolling.maxSize
  22. spark.executor.logs.rolling.strategy
  23. spark.executor.logs.rolling.time.interval
  24. spark.executor.userClassPathFirst
  25. // spark.executorEnv.[EnvironmentVariableName]
  26. spark.python.profile
  27. spark.python.profile.dump
  28. spark.python.worker.memory
  29. spark.python.worker.reuse
  30. spark.reducer.maxSizeInFlight
  31. // Shuffle Behavior
  32. spark.shuffle.blockTransferService
  33. spark.shuffle.compress
  34. spark.shuffle.consolidateFiles
  35. spark.shuffle.file.buffer
  36. spark.shuffle.io.maxRetries
  37. spark.shuffle.io.numConnectionsPerPeer
  38. spark.shuffle.io.preferDirectBufs
  39. spark.shuffle.io.retryWait
  40. spark.shuffle.manager
  41. spark.shuffle.memoryFraction
  42. spark.shuffle.service.enabled
  43. spark.shuffle.service.port
  44. spark.shuffle.sort.bypassMergeThreshold
  45. spark.shuffle.spill
  46. spark.shuffle.spill.compress
  47. // Spark UI
  48. spark.eventLog.compress
  49. spark.eventLog.dir
  50. spark.eventLog.enabled
  51. spark.ui.killEnabled
  52. spark.ui.port
  53. spark.ui.retainedJobs
  54. spark.ui.retainedStages
  55. spark.worker.ui.retainedExecutors
  56. spark.worker.ui.retainedDrivers
  57. // Compression and Serialization
  58. spark.broadcast.compress
  59. spark.closure.serializer
  60. spark.io.compression.codec
  61. spark.io.compression.lz4.blockSize
  62. spark.io.compression.snappy.blockSize
  63. spark.kryo.classesToRegister
  64. spark.kryo.referenceTracking
  65. spark.kryo.registrationRequired
  66. spark.kryo.registrator
  67. spark.kryoserializer.buffer.max
  68. spark.kryoserializer.buffer
  69. spark.rdd.compress
  70. spark.serializer
  71. spark.serializer.objectStreamReset
  72. // Execution Behavior
  73. spark.broadcast.blockSize
  74. spark.broadcast.factory
  75. spark.cleaner.ttl
  76. spark.executor.cores
  77. spark.totalExecutor.cores
  78. spark.default.parallelism
  79. spark.executor.heartbeatInterval
  80. spark.files.fetchTimeout
  81. spark.files.useFetchCache
  82. spark.files.overwrite
  83. spark.hadoop.cloneConf
  84. spark.hadoop.validateOutputSpecs
  85. spark.storage.memoryFraction
  86. spark.storage.memoryMapThreshold
  87. spark.storage.unrollFraction
  88. spark.externalBlockStore.blockManager
  89. spark.externalBlockStore.baseDir
  90. spark.externalBlockStore.url
  91. // Networking
  92. // Dynamic Allocation
  93. spark.dynamicAllocation.enabled
  94. spark.dynamicAllocation.executorIdleTimeout
  95. spark.dynamicAllocation.cachedExecutorIdleTimeout
  96. spark.dynamicAllocation.initialExecutors
  97. spark.dynamicAllocation.maxExecutors
  98. spark.dynamicAllocation.minExecutors
  99. spark.dynamicAllocation.schedulerBacklogTimeout
  100. spark.dynamicAllocation.sustainedSchedulerBacklogTimeout
  101. // Security
  102. // Encryption
  103. // Spark Streaming
  104. spark.streaming.backpressure.enabled
  105. spark.streaming.blockInterval
  106. spark.streaming.receiver.maxRate
  107. spark.streaming.receiver.writeAheadLog.enable
  108. spark.streaming.unpersist
  109. spark.streaming.kafka.maxRatePerPartition
  110. spark.streaming.kafka.maxRetries
  111. spark.streaming.ui.retainedBatches
  112. // SparkR
  113. spark.r.numRBackendThreads