Эх сурвалжийг харах

HUE-2853 [livy] Update Livy to work with Spark 1.5

Spark 1.4+ has moved the python files out from the jar files
and into a pyspark.zip file, which is found in the SPARK_HOME
directory.

This also downgrades json4s to a version compatible with Spark 1.5.
Erick Tryzelaar 10 жил өмнө
parent
commit
8cf5ec6

+ 2 - 0
apps/spark/java/README.rst

@@ -68,6 +68,7 @@ Livy`_. Then run:
 .. code:: shell
 
     % cd $HUE_HOME/apps/spark/java
+    % export SPARK_HOME=/path/to/where/spark/is/installed
     % mvn test
 
 
@@ -78,6 +79,7 @@ In order to run Livy with local sessions, start the server with:
 
 .. code:: shell
 
+    % export SPARK_HOME=/path/to/where/spark/is/installed
     % ./bin/livy-server
 
 Or with YARN sessions by running:

+ 6 - 2
apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/python/PythonInterpreter.scala

@@ -52,7 +52,7 @@ object PythonInterpreter {
     ))
 
     val env = builder.environment()
-    env.put("PYTHONPATH", pythonPath.mkString(File.pathSeparator))
+    env.put("PYTHONPATH", pythonPath)
     env.put("PYTHONUNBUFFERED", "YES")
     env.put("PYSPARK_GATEWAY_PORT", "" + gatewayServer.getListeningPort)
     env.put("SPARK_HOME", sys.env.getOrElse("SPARK_HOME", "."))
@@ -66,8 +66,12 @@ object PythonInterpreter {
 
   private def pythonPath = {
     val pythonPath = new ArrayBuffer[String]
+    for (sparkHome <- sys.env.get("SPARK_HOME")) {
+      pythonPath += Seq(sparkHome, "python", "lib", "pyspark.zip").mkString(File.separator)
+      pythonPath += Seq(sparkHome, "python", "lib", "py4j-0.8.2.1-src.zip").mkString(File.separator)
+    }
     pythonPath ++= Utils.jarOfClass(classOf[SparkContext])
-    pythonPath
+    pythonPath.mkString(File.pathSeparator)
   }
 
   private def createFakeShell(): File = {

+ 1 - 1
apps/spark/java/pom.xml

@@ -49,7 +49,7 @@
     <properties>
         <guava.version>11.0.2</guava.version>
         <jetty.version>9.2.10.v20150310</jetty.version>
-        <json4s.version>3.2.11</json4s.version>
+        <json4s.version>3.2.5</json4s.version>
         <logback.version>1.1.2</logback.version>
         <scala.version>2.10.4</scala.version>
         <scala.binary.version>2.10</scala.binary.version>

+ 1 - 1
maven/pom.xml

@@ -39,7 +39,7 @@
   <properties>
     <hadoop-mr1.version>2.6.0-mr1-cdh5.5.0-SNAPSHOT</hadoop-mr1.version>
     <hadoop.version>2.6.0-cdh5.5.0-SNAPSHOT</hadoop.version>
-    <spark.version>1.3.0-cdh5.5.0-SNAPSHOT</spark.version>
+    <spark.version>1.5.0-cdh5.5.0-SNAPSHOT</spark.version>
     <slf4j.version>1.6.1</slf4j.version>
     <commons-logging.version>1.0.4</commons-logging.version>
     <thrift.version>0.9.0</thrift.version>