Pārlūkot izejas kodu

[livy] Allow to configure the Total Executor Cores in standalone mode

https://github.com/cloudera/hue/pull/276

When submitting snippets, we can't fix the cluster's executor cores flexibly by "Executor cores" with spark cluster in standalone mode. so adding "Total Executor Cores" property.
bwang 10 gadi atpakaļ
vecāks
revīzija
db8057d

+ 33 - 31
apps/spark/java/README.rst

@@ -381,37 +381,39 @@ Creates a new interative Scala, Python or R shell in the cluster.
 Request Body
 ^^^^^^^^^^^^
 
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| name           | description                                                                    | type            |
-+================+================================================================================+=================+
-| kind           | The session kind (required)                                                    | `session kind`_ |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| proxyUser      | The user to impersonate that will run this session (e.g. bob)                  | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| jars           | Files to be placed on the java classpath                                       | list of paths   |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| pyFiles        | Files to be placed on the PYTHONPATH                                           | list of paths   |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| files          | Files to be placed in executor working directory                               | list of paths   |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| driverMemory   | Memory for driver (e.g. 1000M, 2G)                                             | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| driverCores    | Number of cores used by driver (YARN mode only)                                | int             |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| executorMemory | Memory for executor (e.g. 1000M, 2G)                                           | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| executorCores  | Number of cores used by executor                                               | int             |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| numExecutors   | Number of executors (YARN mode only)                                           | int             |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| archives       | Archives to be uncompressed in the executor working directory (YARN mode only) | list of paths   |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| queue          | The YARN queue to submit too (YARN mode only)                                  | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| name           | Name of the application                                                        | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| conf           | Spark configuration property                                                   | Map of key=val  |
-+----------------+--------------------------------------------------------------------------------+-----------------+
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| name              | description                                                                    | type            |
++===================+================================================================================+=================+
+| kind              | The session kind (required)                                                    | `session kind`_ |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| proxyUser         | The user to impersonate that will run this session (e.g. bob)                  | string          |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| jars              | Files to be placed on the java classpath                                       | list of paths   |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| pyFiles           | Files to be placed on the PYTHONPATH                                           | list of paths   |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| files             | Files to be placed in executor working directory                               | list of paths   |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| driverMemory      | Memory for driver (e.g. 1000M, 2G)                                             | string          |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| driverCores       | Number of cores used by driver (YARN mode only)                                | int             |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| executorMemory    | Memory for executor (e.g. 1000M, 2G)                                           | string          |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| executorCores     | Number of cores used by executor                                               | int             |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| totalExecutorCores| number of cluster cores used by executor (Standalone mode only)                | int             |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| numExecutors      | Number of executors (YARN mode only)                                           | int             |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| archives          | Archives to be uncompressed in the executor working directory (YARN mode only) | list of paths   |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| queue             | The YARN queue to submit too (YARN mode only)                                  | string          |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| name              | Name of the application                                                        | string          |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| conf              | Spark configuration property                                                   | Map of key=val  |
++-------------------+--------------------------------------------------------------------------------+-----------------+
 
 
 Response Body

+ 1 - 0
apps/spark/java/conf/spark-user-configurable-options.template

@@ -79,6 +79,7 @@ spark.broadcast.blockSize
 spark.broadcast.factory
 spark.cleaner.ttl
 spark.executor.cores
+spark.totalExecutor.cores
 spark.default.parallelism
 spark.executor.heartbeatInterval
 spark.files.fetchTimeout

+ 1 - 0
apps/spark/java/livy-spark/src/main/resources/com/cloudera/hue/livy/spark/default-spark-user-configurable-options.conf

@@ -78,6 +78,7 @@ spark.broadcast.blockSize
 spark.broadcast.factory
 spark.cleaner.ttl
 spark.executor.cores
+spark.totalExecutor.cores
 spark.default.parallelism
 spark.executor.heartbeatInterval
 spark.files.fetchTimeout

+ 8 - 0
apps/spark/java/livy-spark/src/main/scala/com/cloudera/hue/livy/spark/SparkProcessBuilder.scala

@@ -167,6 +167,14 @@ class SparkProcessBuilder(livyConf: LivyConf, userConfigurableOptions: Set[Strin
     conf("spark.executor.cores", executorCores)
   }
 
+  def totalExecutorCores(totalExecutorCores: Int): SparkProcessBuilder = {
+    this.totalExecutorCores(totalExecutorCores.toString)
+  }
+
+  def totalExecutorCores(totalExecutorCores: String): SparkProcessBuilder = {
+    conf("spark.totalExecutor.cores", totalExecutorCores)
+  }
+
   def executorMemory(executorMemory: String): SparkProcessBuilder = {
     conf("spark.executor.memory", executorMemory)
   }

+ 1 - 0
apps/spark/java/livy-spark/src/main/scala/com/cloudera/hue/livy/spark/batch/BatchSessionFactory.scala

@@ -50,6 +50,7 @@ abstract class BatchSessionFactory(factory: SparkProcessBuilderFactory) extends
     request.driverCores.foreach(builder.driverCores)
     request.executorMemory.foreach(builder.executorMemory)
     request.executorCores.foreach(builder.executorCores)
+    request.totalExecutorCores.foreach(builder.totalExecutorCores)
     request.numExecutors.foreach(builder.numExecutors)
     request.archives.map(RelativePath).foreach(builder.archive)
     request.queue.foreach(builder.queue)

+ 1 - 0
apps/spark/java/livy-spark/src/main/scala/com/cloudera/hue/livy/spark/batch/CreateBatchRequest.scala

@@ -30,6 +30,7 @@ case class CreateBatchRequest(
     driverCores: Option[Int] = None,
     executorMemory: Option[String] = None,
     executorCores: Option[Int] = None,
+    totalExecutorCores: Option[Int] = None,
     numExecutors: Option[Int] = None,
     archives: List[String] = List(),
     queue: Option[String] = None,

+ 1 - 0
apps/spark/java/livy-spark/src/main/scala/com/cloudera/hue/livy/spark/interactive/CreateInteractiveRequest.scala

@@ -30,6 +30,7 @@ case class CreateInteractiveRequest(
     driverCores: Option[Int] = None,
     executorMemory: Option[String] = None,
     executorCores: Option[Int] = None,
+    totalExecutorCores: Option[Int] = None,
     numExecutors: Option[Int] = None,
     archives: List[String] = List(),
     queue: Option[String] = None,

+ 1 - 0
apps/spark/java/livy-spark/src/main/scala/com/cloudera/hue/livy/spark/interactive/InteractiveSessionFactory.scala

@@ -69,6 +69,7 @@ abstract class InteractiveSessionFactory(processFactory: SparkProcessBuilderFact
     request.driverCores.foreach(builder.driverCores)
     request.driverMemory.foreach(builder.driverMemory)
     request.executorCores.foreach(builder.executorCores)
+    request.totalExecutorCores.foreach(builder.totalExecutorCores)
     request.executorMemory.foreach(builder.executorMemory)
     request.numExecutors.foreach(builder.numExecutors)
     request.files.map(RelativePath).foreach(builder.file)

+ 1 - 0
desktop/libs/notebook/src/notebook/connectors/spark_shell.py

@@ -52,6 +52,7 @@ class SparkApi(Api):
     {'name': 'driverCores', 'nice_name': _('Driver Cores'), 'default': '1', 'type': 'number', 'is_yarn': True},
     {'name': 'executorMemory', 'nice_name': _('Executors Memory'), 'default': '1', 'type': 'jvm', 'is_yarn': True},
     {'name': 'executorCores', 'nice_name': _('Executor Cores'), 'default': '1', 'type': 'number', 'is_yarn': True},
+    {'name': 'totalExecutorCores', 'nice_name': _('Total Executor Cores'), 'default': '1', 'type': 'number', 'is_yarn': True},
     {'name': 'queue', 'nice_name': _('Queue'), 'default': '1', 'type': 'string', 'is_yarn': True},
     {'name': 'archives', 'nice_name': _('Archives'), 'default': '', 'type': 'csv-hdfs-files', 'is_yarn': True},
     {'name': 'numExecutors', 'nice_name': _('Executors Numbers'), 'default': '1', 'type': 'number', 'is_yarn': True},