spark-user-configurable-options.template 3.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123
  1. // Application properties
  2. spark.app.name
  3. spark.driver.cores
  4. spark.driver.maxResultSize
  5. spark.driver.memory
  6. spark.executor.memory
  7. spark.extraListeners
  8. // spark.local.dir
  9. spark.logConf
  10. // spark.master
  11. // Runtime Environment
  12. spark.driver.extraClassPath
  13. // spark.driver.extraJavaOptions
  14. spark.driver.extraLibraryPath
  15. spark.driver.userClassPathFirst
  16. spark.executor.extraClassPath
  17. // spark.executor.extraJavaOptions
  18. spark.executor.extraLibraryPath
  19. spark.executor.instances
  20. spark.executor.logs.rolling.maxRetainedFiles
  21. spark.executor.logs.rolling.maxSize
  22. spark.executor.logs.rolling.strategy
  23. spark.executor.logs.rolling.time.interval
  24. spark.executor.userClassPathFirst
  25. // spark.executorEnv.[EnvironmentVariableName]
  26. spark.python.profile
  27. spark.python.profile.dump
  28. spark.python.worker.memory
  29. spark.python.worker.reuse
  30. spark.reducer.maxSizeInFlight
  31. // Shuffle Behavior
  32. spark.shuffle.blockTransferService
  33. spark.shuffle.compress
  34. spark.shuffle.consolidateFiles
  35. spark.shuffle.file.buffer
  36. spark.shuffle.io.maxRetries
  37. spark.shuffle.io.numConnectionsPerPeer
  38. spark.shuffle.io.preferDirectBufs
  39. spark.shuffle.io.retryWait
  40. spark.shuffle.manager
  41. spark.shuffle.memoryFraction
  42. spark.shuffle.service.enabled
  43. spark.shuffle.service.port
  44. spark.shuffle.sort.bypassMergeThreshold
  45. spark.shuffle.spill
  46. spark.shuffle.spill.compress
  47. // Spark UI
  48. spark.eventLog.compress
  49. spark.eventLog.dir
  50. spark.eventLog.enabled
  51. spark.ui.killEnabled
  52. spark.ui.port
  53. spark.ui.retainedJobs
  54. spark.ui.retainedStages
  55. spark.worker.ui.retainedExecutors
  56. spark.worker.ui.retainedDrivers
  57. // Compression and Serialization
  58. spark.broadcast.compress
  59. spark.closure.serializer
  60. spark.io.compression.codec
  61. spark.io.compression.lz4.blockSize
  62. spark.io.compression.snappy.blockSize
  63. spark.kryo.classesToRegister
  64. spark.kryo.referenceTracking
  65. spark.kryo.registrationRequired
  66. spark.kryo.registrator
  67. spark.kryoserializer.buffer.max
  68. spark.kryoserializer.buffer
  69. spark.rdd.compress
  70. spark.serializer
  71. spark.serializer.objectStreamReset
  72. // Execution Behavior
  73. spark.broadcast.blockSize
  74. spark.broadcast.factory
  75. spark.cleaner.ttl
  76. spark.executor.cores
  77. spark.default.parallelism
  78. spark.executor.heartbeatInterval
  79. spark.files.fetchTimeout
  80. spark.files.useFetchCache
  81. spark.files.overwrite
  82. spark.hadoop.cloneConf
  83. spark.hadoop.validateOutputSpecs
  84. spark.storage.memoryFraction
  85. spark.storage.memoryMapThreshold
  86. spark.storage.unrollFraction
  87. spark.externalBlockStore.blockManager
  88. spark.externalBlockStore.baseDir
  89. spark.externalBlockStore.url
  90. // Networking
  91. // Dynamic Allocation
  92. spark.dynamicAllocation.enabled
  93. spark.dynamicAllocation.executorIdleTimeout
  94. spark.dynamicAllocation.cachedExecutorIdleTimeout
  95. spark.dynamicAllocation.initialExecutors
  96. spark.dynamicAllocation.maxExecutors
  97. spark.dynamicAllocation.minExecutors
  98. spark.dynamicAllocation.schedulerBacklogTimeout
  99. spark.dynamicAllocation.sustainedSchedulerBacklogTimeout
  100. // Security
  101. // Encryption
  102. // Spark Streaming
  103. spark.streaming.backpressure.enabled
  104. spark.streaming.blockInterval
  105. spark.streaming.receiver.maxRate
  106. spark.streaming.receiver.writeAheadLog.enable
  107. spark.streaming.unpersist
  108. spark.streaming.kafka.maxRatePerPartition
  109. spark.streaming.kafka.maxRetries
  110. spark.streaming.ui.retainedBatches
  111. // SparkR
  112. spark.r.numRBackendThreads