Ver Fonte

[livy] Add support for specifying interactive memory and core limits

Erick Tryzelaar há 10 anos atrás
pai
commit
a415ae0

+ 5 - 1
apps/spark/java/livy-server/src/main/scala/com/cloudera/hue/livy/server/interactive/InteractiveSessionManager.scala

@@ -98,7 +98,11 @@ class SessionManager(factory: InteractiveSessionFactory) extends Logging {
 }
 
 case class CreateInteractiveRequest(kind: Kind,
-                                    proxyUser: Option[String] = None)
+                                    proxyUser: Option[String] = None,
+                                    driverMemory: Option[String] = None,
+                                    driverCores: Option[Int] = None,
+                                    executorMemory: Option[String] = None,
+                                    executorCores: Option[Int] = None)
 
 class SessionNotFound extends Exception
 

+ 5 - 0
apps/spark/java/livy-server/src/main/scala/com/cloudera/hue/livy/server/interactive/InteractiveSessionProcess.scala

@@ -45,6 +45,11 @@ object InteractiveSessionProcess extends Logging {
 
     val builder = new SparkSubmitProcessBuilder(livyConf)
 
+    createInteractiveRequest.driverCores.foreach(builder.driverCores)
+    createInteractiveRequest.driverMemory.foreach(builder.driverMemory)
+    createInteractiveRequest.executorCores.foreach(builder.executorCores)
+    createInteractiveRequest.executorMemory.foreach(builder.executorMemory)
+
     builder.className("com.cloudera.hue.livy.repl.Main")
 
     sys.env.get("LIVY_REPL_JAVA_OPTS").foreach(builder.driverJavaOptions)

+ 4 - 0
apps/spark/java/livy-server/src/main/scala/com/cloudera/hue/livy/server/interactive/InteractiveSessionYarn.scala

@@ -46,6 +46,10 @@ object InteractiveSessionYarn {
     builder.className("com.cloudera.hue.livy.repl.Main")
     builder.driverJavaOptions(f"-Dlivy.repl.callback-url=$url -Dlivy.repl.port=0")
     createInteractiveRequest.proxyUser.foreach(builder.proxyUser)
+    createInteractiveRequest.driverMemory.foreach(builder.driverMemory)
+    createInteractiveRequest.driverCores.foreach(builder.driverCores)
+    createInteractiveRequest.executorMemory.foreach(builder.executorMemory)
+    createInteractiveRequest.executorCores.foreach(builder.executorCores)
 
     builder.redirectOutput(Redirect.PIPE)
     builder.redirectErrorStream(redirect = true)