Эх сурвалжийг харах

[livy] Don't bake in a path to fake_shell.py, pass along SPARK_HOME, set SPARK_CONF_DIR

Erick Tryzelaar 10 жил өмнө
parent
commit
551ce40

+ 5 - 3
apps/spark/java/livy-repl/src/main/resources/fake_pyspark.sh

@@ -7,8 +7,10 @@ if [ -z "$SPARK_HOME" ]; then
 	exit 1
 fi
 
-source "$SPARK_HOME"/bin/utils.sh
-source "$SPARK_HOME"/bin/load-spark-env.sh
+export SPARK_CONF_DIR="$SPARK_HOME/conf"
+
+source "$SPARK_HOME/bin/utils.sh"
+source "$SPARK_HOME/bin/load-spark-env.sh"
 
 export PYTHONPATH="$SPARK_HOME/python/:$PYTHONPATH"
 
@@ -19,4 +21,4 @@ done
 export OLD_PYTHONSTARTUP="$PYTHONSTARTUP"
 export PYTHONSTARTUP="$SPARK_HOME/python/pyspark/shell.py"
 
-exec python livy-repl/src/main/resources/fake_shell.py
+exec python "$@"

+ 6 - 0
apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/Client.scala

@@ -104,6 +104,12 @@ class Client(yarnConf: YarnConfiguration) {
       containerCtx.getEnvironment()("MASTER") = master
     }
 
+    // FIXME: Spark needs the `SPARK_HOME` environment passed through to run on YARN. This needs a better approach.
+    val spark_home = System.getenv("SPARK_HOME")
+    if (spark_home != null) {
+      containerCtx.getEnvironment()("SPARK_HOME") = spark_home
+    }
+
     appContext.setApplicationId(appId)
     appContext.setAMContainerSpec(containerCtx)
     appContext.setApplicationType("livy")