Browse Source

[spark] Get the spark repl to run inside yarn (!)

Erick Tryzelaar 11 years ago
parent
commit
7d03aae

+ 2 - 2
apps/spark/java/livy-repl/src/main/scala/Scalatra.scala

@@ -1,10 +1,10 @@
 import javax.servlet.ServletContext
 
+import com.cloudera.hue.livy.repl.LivyApp
 import com.cloudera.hue.livy.repl.interpreter.SparkInterpreter
-import com.cloudera.hue.livy.repl.webapp.LivyApp
 import org.scalatra.LifeCycle
 
-class ScalatraBootstrap extends LifeCycle {
+class ScalatraBootstrap2 extends LifeCycle {
 
   //val system = ActorSystem()
   val sparkInterpreter = new SparkInterpreter

+ 1 - 1
apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/WebApp.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.livy.repl.webapp
+package com.cloudera.hue.livy.repl
 
 import akka.util.Timeout
 import com.cloudera.hue.livy.repl.interpreter.SparkInterpreter

+ 4 - 2
apps/spark/java/livy-yarn/pom.xml

@@ -72,6 +72,7 @@
             <version>${scala.version}</version>
         </dependency>
 
+        <!--
         <dependency>
             <groupId>org.scalatra</groupId>
             <artifactId>scalatra_2.10</artifactId>
@@ -83,6 +84,7 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        -->
 
         <dependency>
             <groupId>org.eclipse.jetty</groupId>
@@ -90,23 +92,23 @@
             <version>8.1.14.v20131031</version>
         </dependency>
 
-        <!--
         <dependency>
             <groupId>com.cloudera.hue.livy</groupId>
             <artifactId>livy-repl</artifactId>
             <version>${project.version}</version>
             <exclusions>
+                <!--
                 <exclusion>
                     <groupId>org.scala-lang</groupId>
                     <artifactId>scala-compiler</artifactId>
                 </exclusion>
+                -->
                 <exclusion>
                     <groupId>org.slf4j</groupId>
                     <artifactId>slf4j-api</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
-        -->
 
     </dependencies>
 

+ 1 - 1
apps/spark/java/livy-yarn/src/main/bash/run-class.sh

@@ -36,7 +36,7 @@ function check_and_enable_64_bit_mode {
 # Check if log4j configuration is specified. If not - set to lib/log4j.xml
 [[ $JAVA_OPTS != *-Dlog4j.configuration* && -f $DEFAULT_LOG4J_FILE ]] && JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$DEFAULT_LOG4J_FILE"
 
-JAVA_OPTS="$JAVA_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006"
+JAVA_OPTS="$JAVA_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006"
 
 echo $JAVA $JAVA_OPTS -cp "$CLASSPATH" "$@"
 exec $JAVA $JAVA_OPTS -cp "$CLASSPATH" "$@"

+ 8 - 2
apps/spark/java/livy-yarn/src/main/scala/Scalatra.scala

@@ -1,14 +1,20 @@
 import javax.servlet.ServletContext
 
-import com.cloudera.hue.livy.yarn.WebApp
+import com.cloudera.hue.livy.repl.interpreter.SparkInterpreter
+import com.cloudera.hue.livy.repl.LivyApp
 import org.scalatra.LifeCycle
 
 class ScalatraBootstrap extends LifeCycle {
 
+  //val system = ActorSystem()
+  val sparkInterpreter = new SparkInterpreter
+
   override def init(context: ServletContext): Unit = {
-    context.mount(new WebApp, "/*")
+    context.mount(new LivyApp(sparkInterpreter), "/*")
   }
 
   override def destroy(context: ServletContext): Unit = {
+    sparkInterpreter.close()
+    //system.shutdown()
   }
 }

+ 1 - 1
apps/spark/java/pom.xml

@@ -55,8 +55,8 @@
         <!--
         <module>livy-assembly</module>
         <module>livy-server</module>
-        <module>livy-repl</module>
         -->
+        <module>livy-repl</module>
         <module>livy-yarn</module>
     </modules>