瀏覽代碼

[livy] Require SPARK_HOME env var to be set

Erick Tryzelaar 10 年之前
父節點
當前提交
6f5c2e2

+ 11 - 1
apps/spark/java/livy-core/src/main/scala/com/cloudera/hue/livy/LivyConf.scala

@@ -18,12 +18,14 @@
 
 package com.cloudera.hue.livy
 
+import java.io.File
 import java.util.concurrent.ConcurrentHashMap
 
 import scala.collection.JavaConverters._
 
 object LivyConf {
   val SESSION_FACTORY_KEY = "livy.server.session.factory"
+  val SPARK_HOME_KEY = "livy.server.spark-home"
   val SPARK_SUBMIT_KEY = "livy.server.spark-submit"
   val IMPERSONATION_ENABLED_KEY = "livy.impersonation.enabled"
 
@@ -94,7 +96,15 @@ class LivyConf(loadDefaults: Boolean) {
   /** Return if the configuration includes this setting */
   def contains(key: String): Boolean = settings.containsKey(key)
 
-  def sparkSubmit(): String = getOption(SPARK_SUBMIT_KEY).getOrElse("spark-submit")
+  /** Return the location of the spark home directory */
+  def sparkHome(): Option[String] = getOption(SPARK_HOME_KEY).orElse(sys.env.get("SPARK_HOME"))
+
+  /** Return the path to the spark-submit executable. */
+  def sparkSubmit(): String = {
+    getOption(SPARK_SUBMIT_KEY)
+      .orElse { sparkHome().map { _ + File.separator + "bin" + File.separator + "spark-submit" } }
+      .getOrElse("spark-submit")
+  }
 
   def sessionKind(): SessionKind = getOption(SESSION_FACTORY_KEY).getOrElse("process") match {
     case "process" => Process()

+ 21 - 1
apps/spark/java/livy-server/src/main/scala/com/cloudera/hue/livy/server/Main.scala

@@ -18,7 +18,8 @@
 
 package com.cloudera.hue.livy.server
 
-import java.io.IOException
+import java.io.{File, IOException}
+import java.nio.file.{Paths, Files}
 import javax.servlet.ServletContext
 
 import com.cloudera.hue.livy._
@@ -45,6 +46,7 @@ object Main {
     val port = livyConf.getInt("livy.server.port", 8998)
 
     // Make sure the `spark-submit` program exists, otherwise much of livy won't work.
+    testSparkHome(livyConf)
     testSparkSubmit(livyConf)
 
     val server = new WebServer(host, port)
@@ -66,6 +68,23 @@ object Main {
     }
   }
 
+  /**
+   * Sets the spark-submit path if it's not configured in the LivyConf
+   */
+  private def testSparkHome(livyConf: LivyConf) = {
+    val sparkHome = livyConf.sparkHome().getOrElse {
+      System.err.println("Livy requires the SPARK_HOME environment variable")
+      sys.exit(1)
+    }
+
+    val sparkHomeFile = new File(sparkHome)
+
+    if (!sparkHomeFile.exists) {
+      System.err.println("SPARK_HOME path does not exist")
+      sys.exit(1)
+    }
+  }
+
   /**
    * Test that the configured `spark-submit` executable exists.
    *
@@ -121,6 +140,7 @@ object Main {
       case _ => throw new IOException(f"Unable to determing spark-submit version [$exitCode]:\n$output")
     }
   }
+
 }
 
 class ScalatraBootstrap