Przeglądaj źródła

[livy] Allow the spark-submit to be configured

Erick Tryzelaar 10 lat temu
rodzic
commit
c08a961

+ 4 - 0
apps/spark/java/conf/livy-defaults.conf.tmpl

@@ -8,6 +8,10 @@
 # What session factory to use. The options are `thread`, `process`, and `yarn`.
 # livy.server.session.factory = process
 
+# What spark-submit executable path to use to submit spark applications. Defaults to
+# `spark-submit`.
+# livy.server.spark-submit = spark-submit
+
 # Location to find the livy assembly. If not specified, livy will determine the
 # assembly from the local jarfile. If using `yarn` sessions, this may be on HDFS.
 # livy.yarn.jar = hdfs://localhost:8020/user/hue/share/lib/livy-assembly.jar

+ 3 - 0
apps/spark/java/livy-core/src/main/scala/com/cloudera/hue/livy/LivyConf.scala

@@ -24,6 +24,7 @@ import scala.collection.JavaConverters._
 
 object LivyConf {
   val SESSION_FACTORY_KEY = "livy.server.session.factory"
+  val SPARK_SUBMIT_KEY = "livy.server.spark-submit"
 
   sealed trait SessionKind
   case class Thread() extends SessionKind
@@ -90,6 +91,8 @@ class LivyConf(loadDefaults: Boolean) {
   /** Return if the configuration includes this setting */
   def contains(key: String): Boolean = settings.containsKey(key)
 
+  def sparkSubmit(): String = getOption(SPARK_SUBMIT_KEY).getOrElse("spark-submit")
+
   def sessionKind(): SessionKind = getOption(SESSION_FACTORY_KEY).getOrElse("process") match {
     case "process" => Process()
     case "thread" => Thread()

+ 1 - 1
apps/spark/java/livy-core/src/main/scala/com/cloudera/hue/livy/spark/SparkSubmitProcessBuilder.scala

@@ -42,7 +42,7 @@ class SparkSubmitProcessBuilder(livyConf: LivyConf) extends Logging {
 
   private[this] val fsRoot = livyConf.filesystemRoot()
 
-  private[this] var _executable: Path = AbsolutePath("spark-submit")
+  private[this] var _executable: Path = AbsolutePath(livyConf.sparkSubmit())
   private[this] var _master: Option[String] = None
   private[this] var _deployMode: Option[String] = None
   private[this] var _className: Option[String] = None