Explorar o código

[livy] Allow to configure the Total Executor Cores in standalone mode

https://github.com/cloudera/hue/pull/276

When submitting snippets, we can't fix the cluster's executor cores flexibly by "Executor cores" with spark cluster in standalone mode. so adding "Total Executor Cores" property.
bwang %!s(int64=10) %!d(string=hai) anos
pai
achega
db8057d

+ 33 - 31
apps/spark/java/README.rst

@@ -381,37 +381,39 @@ Creates a new interative Scala, Python or R shell in the cluster.
 Request Body
 ^^^^^^^^^^^^
 
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| name           | description                                                                    | type            |
-+================+================================================================================+=================+
-| kind           | The session kind (required)                                                    | `session kind`_ |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| proxyUser      | The user to impersonate that will run this session (e.g. bob)                  | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| jars           | Files to be placed on the java classpath                                       | list of paths   |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| pyFiles        | Files to be placed on the PYTHONPATH                                           | list of paths   |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| files          | Files to be placed in executor working directory                               | list of paths   |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| driverMemory   | Memory for driver (e.g. 1000M, 2G)                                             | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| driverCores    | Number of cores used by driver (YARN mode only)                                | int             |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| executorMemory | Memory for executor (e.g. 1000M, 2G)                                           | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| executorCores  | Number of cores used by executor                                               | int             |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| numExecutors   | Number of executors (YARN mode only)                                           | int             |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| archives       | Archives to be uncompressed in the executor working directory (YARN mode only) | list of paths   |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| queue          | The YARN queue to submit too (YARN mode only)                                  | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| name           | Name of the application                                                        | string          |
-+----------------+--------------------------------------------------------------------------------+-----------------+
-| conf           | Spark configuration property                                                   | Map of key=val  |
-+----------------+--------------------------------------------------------------------------------+-----------------+
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| name              | description                                                                    | type            |
++===================+================================================================================+=================+
+| kind              | The session kind (required)                                                    | `session kind`_ |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| proxyUser         | The user to impersonate that will run this session (e.g. bob)                  | string          |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| jars              | Files to be placed on the java classpath                                       | list of paths   |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| pyFiles           | Files to be placed on the PYTHONPATH                                           | list of paths   |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| files             | Files to be placed in executor working directory                               | list of paths   |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| driverMemory      | Memory for driver (e.g. 1000M, 2G)                                             | string          |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| driverCores       | Number of cores used by driver (YARN mode only)                                | int             |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| executorMemory    | Memory for executor (e.g. 1000M, 2G)                                           | string          |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| executorCores     | Number of cores used by executor                                               | int             |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| totalExecutorCores| number of cluster cores used by executor (Standalone mode only)                | int             |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| numExecutors      | Number of executors (YARN mode only)                                           | int             |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| archives          | Archives to be uncompressed in the executor working directory (YARN mode only) | list of paths   |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| queue             | The YARN queue to submit too (YARN mode only)                                  | string          |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| name              | Name of the application                                                        | string          |
++-------------------+--------------------------------------------------------------------------------+-----------------+
+| conf              | Spark configuration property                                                   | Map of key=val  |
++-------------------+--------------------------------------------------------------------------------+-----------------+
 
 
 Response Body

+ 1 - 0
apps/spark/java/conf/spark-user-configurable-options.template

@@ -79,6 +79,7 @@ spark.broadcast.blockSize
 spark.broadcast.factory
 spark.cleaner.ttl
 spark.executor.cores
+spark.totalExecutor.cores
 spark.default.parallelism
 spark.executor.heartbeatInterval
 spark.files.fetchTimeout

+ 1 - 0
apps/spark/java/livy-spark/src/main/resources/com/cloudera/hue/livy/spark/default-spark-user-configurable-options.conf

@@ -78,6 +78,7 @@ spark.broadcast.blockSize
 spark.broadcast.factory
 spark.cleaner.ttl
 spark.executor.cores
+spark.totalExecutor.cores
 spark.default.parallelism
 spark.executor.heartbeatInterval
 spark.files.fetchTimeout

+ 8 - 0
apps/spark/java/livy-spark/src/main/scala/com/cloudera/hue/livy/spark/SparkProcessBuilder.scala

@@ -167,6 +167,14 @@ class SparkProcessBuilder(livyConf: LivyConf, userConfigurableOptions: Set[Strin
     conf("spark.executor.cores", executorCores)
   }
 
+  def totalExecutorCores(totalExecutorCores: Int): SparkProcessBuilder = {
+    this.totalExecutorCores(totalExecutorCores.toString)
+  }
+
+  def totalExecutorCores(totalExecutorCores: String): SparkProcessBuilder = {
+    conf("spark.totalExecutor.cores", totalExecutorCores)
+  }
+
   def executorMemory(executorMemory: String): SparkProcessBuilder = {
     conf("spark.executor.memory", executorMemory)
   }

+ 1 - 0
apps/spark/java/livy-spark/src/main/scala/com/cloudera/hue/livy/spark/batch/BatchSessionFactory.scala

@@ -50,6 +50,7 @@ abstract class BatchSessionFactory(factory: SparkProcessBuilderFactory) extends
     request.driverCores.foreach(builder.driverCores)
     request.executorMemory.foreach(builder.executorMemory)
     request.executorCores.foreach(builder.executorCores)
+    request.totalExecutorCores.foreach(builder.totalExecutorCores)
     request.numExecutors.foreach(builder.numExecutors)
     request.archives.map(RelativePath).foreach(builder.archive)
     request.queue.foreach(builder.queue)

+ 1 - 0
apps/spark/java/livy-spark/src/main/scala/com/cloudera/hue/livy/spark/batch/CreateBatchRequest.scala

@@ -30,6 +30,7 @@ case class CreateBatchRequest(
     driverCores: Option[Int] = None,
     executorMemory: Option[String] = None,
     executorCores: Option[Int] = None,
+    totalExecutorCores: Option[Int] = None,
     numExecutors: Option[Int] = None,
     archives: List[String] = List(),
     queue: Option[String] = None,

+ 1 - 0
apps/spark/java/livy-spark/src/main/scala/com/cloudera/hue/livy/spark/interactive/CreateInteractiveRequest.scala

@@ -30,6 +30,7 @@ case class CreateInteractiveRequest(
     driverCores: Option[Int] = None,
     executorMemory: Option[String] = None,
     executorCores: Option[Int] = None,
+    totalExecutorCores: Option[Int] = None,
     numExecutors: Option[Int] = None,
     archives: List[String] = List(),
     queue: Option[String] = None,

+ 1 - 0
apps/spark/java/livy-spark/src/main/scala/com/cloudera/hue/livy/spark/interactive/InteractiveSessionFactory.scala

@@ -69,6 +69,7 @@ abstract class InteractiveSessionFactory(processFactory: SparkProcessBuilderFact
     request.driverCores.foreach(builder.driverCores)
     request.driverMemory.foreach(builder.driverMemory)
     request.executorCores.foreach(builder.executorCores)
+    request.totalExecutorCores.foreach(builder.totalExecutorCores)
     request.executorMemory.foreach(builder.executorMemory)
     request.numExecutors.foreach(builder.numExecutors)
     request.files.map(RelativePath).foreach(builder.file)

+ 1 - 0
desktop/libs/notebook/src/notebook/connectors/spark_shell.py

@@ -52,6 +52,7 @@ class SparkApi(Api):
     {'name': 'driverCores', 'nice_name': _('Driver Cores'), 'default': '1', 'type': 'number', 'is_yarn': True},
     {'name': 'executorMemory', 'nice_name': _('Executors Memory'), 'default': '1', 'type': 'jvm', 'is_yarn': True},
     {'name': 'executorCores', 'nice_name': _('Executor Cores'), 'default': '1', 'type': 'number', 'is_yarn': True},
+    {'name': 'totalExecutorCores', 'nice_name': _('Total Executor Cores'), 'default': '1', 'type': 'number', 'is_yarn': True},
     {'name': 'queue', 'nice_name': _('Queue'), 'default': '1', 'type': 'string', 'is_yarn': True},
     {'name': 'archives', 'nice_name': _('Archives'), 'default': '', 'type': 'csv-hdfs-files', 'is_yarn': True},
     {'name': 'numExecutors', 'nice_name': _('Executors Numbers'), 'default': '1', 'type': 'number', 'is_yarn': True},