浏览代码

[spark] Rename Sparker to Livy since we're going to be more than Spark

Erick Tryzelaar 11 年之前
父节点
当前提交
9a923e1
共有 34 个文件被更改,包括 288 次插入312 次删除
  1. 5 5
      apps/spark/java/livy-repl/pom.xml
  2. 0 0
      apps/spark/java/livy-repl/src/main/assembly/dist.xml
  3. 20 0
      apps/spark/java/livy-repl/src/main/scala/Scalatra.scala
  4. 7 7
      apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/Interpreter.scala
  5. 2 2
      apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/Main.scala
  6. 3 3
      apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/WebApp.scala
  7. 5 5
      apps/spark/java/livy-server/pom.xml
  8. 8 8
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/LivyApp.java
  9. 6 0
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/LivyConfiguration.java
  10. 1 1
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/resources/ExecuteStatementRequest.java
  11. 5 5
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/resources/SessionResource.java
  12. 4 4
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/resources/StatementResource.java
  13. 1 1
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/ClosedSessionException.java
  14. 1 1
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/Session.java
  15. 1 1
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/SessionManager.java
  16. 1 1
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/SparkSession.java
  17. 1 1
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/Statement.java
  18. 162 0
      apps/spark/java/livy-yarn/pom.xml
  19. 2 13
      apps/spark/java/livy-yarn/src/main/assembly/dist.xml
  20. 0 0
      apps/spark/java/livy-yarn/src/main/bash/run-am.sh
  21. 0 0
      apps/spark/java/livy-yarn/src/main/bash/run-class.sh
  22. 0 0
      apps/spark/java/livy-yarn/src/main/bash/run-job.sh
  23. 0 0
      apps/spark/java/livy-yarn/src/main/resources/log4j.properties
  24. 16 13
      apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/AppMaster.scala
  25. 3 14
      apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/Client.scala
  26. 1 1
      apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/Logging.scala
  27. 1 1
      apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/YarnJob.scala
  28. 32 16
      apps/spark/java/pom.xml
  29. 0 21
      apps/spark/java/sparker-repl/src/main/scala/Scalatra.scala
  30. 0 6
      apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/SparkerConfiguration.java
  31. 0 0
      apps/spark/java/src/main/assembly/dist3.xml
  32. 0 81
      apps/spark/spark_server.sh
  33. 0 93
      apps/spark/sparker-client.py
  34. 0 8
      apps/spark/sparker-shell

+ 5 - 5
apps/spark/java/sparker-repl/pom.xml → apps/spark/java/livy-repl/pom.xml

@@ -4,13 +4,13 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <modelVersion>4.0.0</modelVersion>
     <parent>
     <parent>
-        <groupId>com.cloudera.hue.sparker</groupId>
-        <artifactId>sparker-main</artifactId>
+        <groupId>com.cloudera.hue.livy</groupId>
+        <artifactId>livy-main</artifactId>
         <relativePath>../pom.xml</relativePath>
         <relativePath>../pom.xml</relativePath>
         <version>3.7.0-SNAPSHOT</version>
         <version>3.7.0-SNAPSHOT</version>
     </parent>
     </parent>
 
 
-    <artifactId>sparker-repl</artifactId>
+    <artifactId>livy-repl</artifactId>
     <packaging>jar</packaging>
     <packaging>jar</packaging>
 
 
     <properties>
     <properties>
@@ -124,7 +124,7 @@
                         <manifest>
                         <manifest>
                             <addClasspath>true</addClasspath>
                             <addClasspath>true</addClasspath>
                             <classpathPrefix>lib/</classpathPrefix>
                             <classpathPrefix>lib/</classpathPrefix>
-                            <mainClass>com.cloudera.hue.sparker.repl.Main</mainClass>
+                            <mainClass>com.cloudera.hue.livy.repl.Main</mainClass>
                         </manifest>
                         </manifest>
                     </archive>
                     </archive>
                 </configuration>
                 </configuration>
@@ -167,7 +167,7 @@
                             <transformers>
                             <transformers>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>com.cloudera.hue.sparker.server.SparkerApp</mainClass>
+                                    <mainClass>com.cloudera.hue.livy.server.LivyApp</mainClass>
                                 </transformer>
                                 </transformer>
                             </transformers>
                             </transformers>
                         </configuration>
                         </configuration>

+ 0 - 0
apps/spark/java/sparker-repl/src/main/assembly/dist.xml → apps/spark/java/livy-repl/src/main/assembly/dist.xml


+ 20 - 0
apps/spark/java/livy-repl/src/main/scala/Scalatra.scala

@@ -0,0 +1,20 @@
+import javax.servlet.ServletContext
+
+import com.cloudera.hue.livy.repl.interpreter.SparkInterpreter
+import com.cloudera.hue.livy.repl.webapp.LivyApp
+import org.scalatra.LifeCycle
+
+class ScalatraBootstrap extends LifeCycle {
+
+  //val system = ActorSystem()
+  val sparkInterpreter = new SparkInterpreter
+
+  override def init(context: ServletContext): Unit = {
+    context.mount(new LivyApp(sparkInterpreter), "/*")
+  }
+
+  override def destroy(context: ServletContext): Unit = {
+    sparkInterpreter.close()
+    //system.shutdown()
+  }
+}

+ 7 - 7
apps/spark/java/sparker-repl/src/main/scala/com/cloudera/hue/sparker/repl/Interpreter.scala → apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/Interpreter.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.repl.interpreter
+package com.cloudera.hue.livy.repl.interpreter
 
 
 import java.io.{BufferedReader, PipedReader, PipedWriter, StringWriter}
 import java.io.{BufferedReader, PipedReader, PipedWriter, StringWriter}
 import java.util.concurrent.{BlockingQueue, SynchronousQueue}
 import java.util.concurrent.{BlockingQueue, SynchronousQueue}
@@ -10,14 +10,14 @@ import scala.tools.nsc.SparkHelper
 import scala.tools.nsc.interpreter.{Formatting, _}
 import scala.tools.nsc.interpreter.{Formatting, _}
 import scala.tools.nsc.util.ClassPath
 import scala.tools.nsc.util.ClassPath
 
 
-class SparkerInterpreter {
+class SparkInterpreter {
   private implicit def executor: ExecutionContext = ExecutionContext.global
   private implicit def executor: ExecutionContext = ExecutionContext.global
 
 
   private var running = false;
   private var running = false;
   private val inQueue = new SynchronousQueue[Request]
   private val inQueue = new SynchronousQueue[Request]
   private val inWriter = new PipedWriter()
   private val inWriter = new PipedWriter()
 
 
-  org.apache.spark.repl.Main.interp = new SparkerILoop(
+  org.apache.spark.repl.Main.interp = new ILoop(
     this,
     this,
     inQueue,
     inQueue,
     new BufferedReader(new PipedReader(inWriter)),
     new BufferedReader(new PipedReader(inWriter)),
@@ -48,13 +48,13 @@ class SparkerInterpreter {
   }
   }
 }
 }
 
 
-class SparkerILoop(parent: SparkerInterpreter, inQueue: BlockingQueue[Request], in0: BufferedReader, outString: StringWriter) extends SparkILoop(in0, new JPrintWriter(outString)) {
+private class ILoop(parent: SparkInterpreter, inQueue: BlockingQueue[Request], in0: BufferedReader, outString: StringWriter) extends SparkILoop(in0, new JPrintWriter(outString)) {
 
 
-  class SparkerILoopInterpreter extends SparkILoopInterpreter {
+  class ILoopInterpreter extends SparkILoopInterpreter {
     outer =>
     outer =>
 
 
     override lazy val formatting = new Formatting {
     override lazy val formatting = new Formatting {
-      def prompt = SparkerILoop.this.prompt
+      def prompt = ILoop.this.prompt
     }
     }
     override protected def parentClassLoader = SparkHelper.explicitParentLoader(settings).getOrElse(classOf[SparkILoop].getClassLoader)
     override protected def parentClassLoader = SparkHelper.explicitParentLoader(settings).getOrElse(classOf[SparkILoop].getClassLoader)
   }
   }
@@ -69,7 +69,7 @@ class SparkerILoop(parent: SparkerInterpreter, inQueue: BlockingQueue[Request],
       settings.classpath.value)((l, r) => ClassPath.join(l, r))
       settings.classpath.value)((l, r) => ClassPath.join(l, r))
     this.settings.classpath.value = totalClassPath
     this.settings.classpath.value = totalClassPath
 
 
-    intp = new SparkerILoopInterpreter
+    intp = new ILoopInterpreter
   }
   }
 
 
   private val replayQuestionMessage =
   private val replayQuestionMessage =

+ 2 - 2
apps/spark/java/sparker-repl/src/main/scala/com/cloudera/hue/sparker/repl/Main.scala → apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/Main.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.repl
+package com.cloudera.hue.livy.repl
 
 
 import org.eclipse.jetty.server.Server
 import org.eclipse.jetty.server.Server
 import org.eclipse.jetty.servlet.{ServletHolder, DefaultServlet}
 import org.eclipse.jetty.servlet.{ServletHolder, DefaultServlet}
@@ -14,7 +14,7 @@ object Main {
     val context = new WebAppContext()
     val context = new WebAppContext()
 
 
     context.setContextPath("/")
     context.setContextPath("/")
-    context.setResourceBase("src/main/com/cloudera/hue/sparker/repl")
+    context.setResourceBase("src/main/com/cloudera/hue/livy/repl")
     context.addEventListener(new ScalatraListener)
     context.addEventListener(new ScalatraListener)
 
 
     context.addServlet(classOf[DefaultServlet], "/")
     context.addServlet(classOf[DefaultServlet], "/")

+ 3 - 3
apps/spark/java/sparker-repl/src/main/scala/com/cloudera/hue/sparker/repl/WebApp.scala → apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/WebApp.scala

@@ -1,14 +1,14 @@
-package com.cloudera.hue.sparker.repl.webapp
+package com.cloudera.hue.livy.repl.webapp
 
 
 import akka.util.Timeout
 import akka.util.Timeout
-import com.cloudera.hue.sparker.repl.interpreter.SparkerInterpreter
+import com.cloudera.hue.livy.repl.interpreter.SparkInterpreter
 import org.json4s.{DefaultFormats, Formats}
 import org.json4s.{DefaultFormats, Formats}
 import org.scalatra.json._
 import org.scalatra.json._
 import org.scalatra.{Accepted, AsyncResult, FutureSupport, ScalatraServlet}
 import org.scalatra.{Accepted, AsyncResult, FutureSupport, ScalatraServlet}
 
 
 import scala.concurrent.{Future, ExecutionContext, ExecutionContextExecutor}
 import scala.concurrent.{Future, ExecutionContext, ExecutionContextExecutor}
 
 
-class SparkerApp(interpreter: SparkerInterpreter) extends ScalatraServlet with FutureSupport with JacksonJsonSupport {
+class LivyApp(interpreter: SparkInterpreter) extends ScalatraServlet with FutureSupport with JacksonJsonSupport {
 
 
   protected implicit def executor: ExecutionContextExecutor = ExecutionContext.global
   protected implicit def executor: ExecutionContextExecutor = ExecutionContext.global
   protected implicit def defaultTimeout: Timeout = Timeout(10)
   protected implicit def defaultTimeout: Timeout = Timeout(10)

+ 5 - 5
apps/spark/java/sparker-server/pom.xml → apps/spark/java/livy-server/pom.xml

@@ -21,13 +21,13 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <modelVersion>4.0.0</modelVersion>
     <parent>
     <parent>
-        <groupId>com.cloudera.hue.sparker</groupId>
-        <artifactId>sparker-main</artifactId>
+        <groupId>com.cloudera.hue.livy</groupId>
+        <artifactId>livy-main</artifactId>
         <relativePath>../pom.xml</relativePath>
         <relativePath>../pom.xml</relativePath>
         <version>3.7.0-SNAPSHOT</version>
         <version>3.7.0-SNAPSHOT</version>
     </parent>
     </parent>
 
 
-    <artifactId>sparker-server</artifactId>
+    <artifactId>livy-server</artifactId>
     <packaging>jar</packaging>
     <packaging>jar</packaging>
 
 
     <properties>
     <properties>
@@ -124,7 +124,7 @@
                             <transformers>
                             <transformers>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>com.cloudera.hue.sparker.server.SparkerApp</mainClass>
+                                    <mainClass>com.cloudera.hue.livy.server.LivyApp</mainClass>
                                 </transformer>
                                 </transformer>
                             </transformers>
                             </transformers>
                         </configuration>
                         </configuration>
@@ -161,7 +161,7 @@
                         <manifest>
                         <manifest>
                             <addClasspath>true</addClasspath>
                             <addClasspath>true</addClasspath>
                             <classpathPrefix>lib/</classpathPrefix>
                             <classpathPrefix>lib/</classpathPrefix>
-                            <mainClass>com.cloudera.hue.sparker.server.SparkerMain</mainClass>
+                            <mainClass>com.cloudera.hue.livy.server.LivyMain</mainClass>
                         </manifest>
                         </manifest>
                     </archive>
                     </archive>
                 </configuration>
                 </configuration>

+ 8 - 8
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/SparkerApp.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/LivyApp.java

@@ -1,8 +1,8 @@
-package com.cloudera.hue.sparker.server;
+package com.cloudera.hue.livy.server;
 
 
-import com.cloudera.hue.sparker.server.resources.StatementResource;
-import com.cloudera.hue.sparker.server.resources.SessionResource;
-import com.cloudera.hue.sparker.server.sessions.SessionManager;
+import com.cloudera.hue.livy.server.sessions.SessionManager;
+import com.cloudera.hue.livy.server.resources.StatementResource;
+import com.cloudera.hue.livy.server.resources.SessionResource;
 import com.sun.jersey.core.spi.factory.ResponseBuilderImpl;
 import com.sun.jersey.core.spi.factory.ResponseBuilderImpl;
 import io.dropwizard.Application;
 import io.dropwizard.Application;
 import io.dropwizard.setup.Bootstrap;
 import io.dropwizard.setup.Bootstrap;
@@ -11,19 +11,19 @@ import io.dropwizard.setup.Environment;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.ext.ExceptionMapper;
 import javax.ws.rs.ext.ExceptionMapper;
 
 
-public class SparkerApp extends Application<SparkerConfiguration> {
+public class LivyApp extends Application<LivyConfiguration> {
 
 
     public static void main(String[] args) throws Exception {
     public static void main(String[] args) throws Exception {
-        new SparkerApp().run(args);
+        new LivyApp().run(args);
     }
     }
 
 
     @Override
     @Override
-    public void initialize(Bootstrap<SparkerConfiguration> bootstrap) {
+    public void initialize(Bootstrap<LivyConfiguration> bootstrap) {
 
 
     }
     }
 
 
     @Override
     @Override
-    public void run(SparkerConfiguration sparkerConfiguration, Environment environment) throws Exception {
+    public void run(LivyConfiguration livyConfiguration, Environment environment) throws Exception {
         final SessionManager sessionManager = new SessionManager();
         final SessionManager sessionManager = new SessionManager();
         environment.jersey().register(new SessionResource(sessionManager));
         environment.jersey().register(new SessionResource(sessionManager));
         environment.jersey().register(new StatementResource(sessionManager));
         environment.jersey().register(new StatementResource(sessionManager));

+ 6 - 0
apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/LivyConfiguration.java

@@ -0,0 +1,6 @@
+package com.cloudera.hue.livy.server;
+
+import io.dropwizard.Configuration;
+
+public class LivyConfiguration extends Configuration {
+}

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/resources/ExecuteStatementRequest.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/resources/ExecuteStatementRequest.java

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.server.resources;
+package com.cloudera.hue.livy.server.resources;
 
 
 import org.hibernate.validator.constraints.NotEmpty;
 import org.hibernate.validator.constraints.NotEmpty;
 
 

+ 5 - 5
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/resources/SessionResource.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/resources/SessionResource.java

@@ -1,9 +1,9 @@
-package com.cloudera.hue.sparker.server.resources;
+package com.cloudera.hue.livy.server.resources;
 
 
-import com.cloudera.hue.sparker.server.sessions.Statement;
-import com.cloudera.hue.sparker.server.sessions.ClosedSessionException;
-import com.cloudera.hue.sparker.server.sessions.Session;
-import com.cloudera.hue.sparker.server.sessions.SessionManager;
+import com.cloudera.hue.livy.server.sessions.ClosedSessionException;
+import com.cloudera.hue.livy.server.sessions.Session;
+import com.cloudera.hue.livy.server.sessions.SessionManager;
+import com.cloudera.hue.livy.server.sessions.Statement;
 import com.codahale.metrics.annotation.Timed;
 import com.codahale.metrics.annotation.Timed;
 import com.sun.jersey.core.spi.factory.ResponseBuilderImpl;
 import com.sun.jersey.core.spi.factory.ResponseBuilderImpl;
 
 

+ 4 - 4
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/resources/StatementResource.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/resources/StatementResource.java

@@ -1,8 +1,8 @@
-package com.cloudera.hue.sparker.server.resources;
+package com.cloudera.hue.livy.server.resources;
 
 
-import com.cloudera.hue.sparker.server.sessions.Session;
-import com.cloudera.hue.sparker.server.sessions.SessionManager;
-import com.cloudera.hue.sparker.server.sessions.Statement;
+import com.cloudera.hue.livy.server.sessions.Session;
+import com.cloudera.hue.livy.server.sessions.SessionManager;
+import com.cloudera.hue.livy.server.sessions.Statement;
 import com.codahale.metrics.annotation.Timed;
 import com.codahale.metrics.annotation.Timed;
 
 
 import javax.ws.rs.*;
 import javax.ws.rs.*;

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/sessions/ClosedSessionException.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/ClosedSessionException.java

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.server.sessions;
+package com.cloudera.hue.livy.server.sessions;
 
 
 public class ClosedSessionException extends Throwable {
 public class ClosedSessionException extends Throwable {
 }
 }

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/sessions/Session.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/Session.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  * limitations under the License.
  */
  */
 
 
-package com.cloudera.hue.sparker.server.sessions;
+package com.cloudera.hue.livy.server.sessions;
 
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/sessions/SessionManager.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/SessionManager.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  * limitations under the License.
  */
  */
 
 
-package com.cloudera.hue.sparker.server.sessions;
+package com.cloudera.hue.livy.server.sessions;
 
 
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.LoggerFactory;

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/sessions/SparkSession.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/SparkSession.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  * limitations under the License.
  */
  */
 
 
-package com.cloudera.hue.sparker.server.sessions;
+package com.cloudera.hue.livy.server.sessions;
 
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectMapper;

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/sessions/Statement.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/Statement.java

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.server.sessions;
+package com.cloudera.hue.livy.server.sessions;
 
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 

+ 162 - 0
apps/spark/java/livy-yarn/pom.xml

@@ -0,0 +1,162 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.cloudera.hue.sparker</groupId>
+        <artifactId>sparker-main</artifactId>
+        <relativePath>../pom.xml</relativePath>
+        <version>3.7.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>sparker-yarn</artifactId>
+    <packaging>jar</packaging>
+
+    <properties>
+        <hadoop.version>2.5.0</hadoop.version>
+        <scala.binary.version>2.10</scala.binary.version>
+        <scala.macros.version>2.0.1</scala.macros.version>
+        <scala.version>2.10.3</scala.version>
+        <PermGen>64m</PermGen>
+        <MaxPermGen>512m</MaxPermGen>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-client</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-api</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+            <version>${scala.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.scala-tools</groupId>
+                <artifactId>maven-scala-plugin</artifactId>
+                <version>2.15.2</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <scalaVersion>${scala.version}</scalaVersion>
+                </configuration
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptor>src/main/assembly/dist.xml</descriptor>
+                </configuration>
+            </plugin>
+
+						<!--
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>1.6</version>
+                <configuration>
+                    <createDependencyReducedPom>true</createDependencyReducedPom>
+                    <filters>
+                        <filter>
+                            <artifact>*:*</artifact>
+                            <excludes>
+                                <exclude>META-INF/*.SF</exclude>
+                                <exclude>META-INF/*.DSA</exclude>
+                                <exclude>META-INF/*.RSA</exclude>
+                            </excludes>
+                        </filter>
+                    </filters>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>com.cloudera.hue.sparker.yarn.Client</mainClass>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+-->
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-dependencies</id>
+                        <phase>prepare-package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
+                            <overWriteReleases>false</overWriteReleases>
+                            <overWriteSnapshots>false</overWriteSnapshots>
+                            <overWriteIfNewer>true</overWriteIfNewer>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <addClasspath>true</addClasspath>
+                            <classpathPrefix>lib/</classpathPrefix>
+                            <maisparkerss>com.cloudera.hue.livy.yarn.Client</mainClass>
+                        </manifest>
+                    </archive>
+                </configuratio>>
+            </plugin>
+
+        </plugins>
+    </build>
+
+    <reporting>
+        <plugins>
+            <plugin>
+                <groupId>org.scala-tools</groupId>
+                <artifactId>maven-scala-plugin</artifactId>
+                <configuration>
+                    <scalaVersion>${scala.version}</scalaVersion>
+                </configuration>
+            </plugin>
+        </plugins>
+    </reporting>
+
+</project>

+ 2 - 13
apps/spark/java/sparker-yarn/src/main/assembly/dist.xml → apps/spark/java/livy-yarn/src/main/assembly/dist.xml

@@ -16,18 +16,13 @@
     <formats>
     <formats>
         <format>tar.gz</format>
         <format>tar.gz</format>
     </formats>
     </formats>
+
     <includeBaseDirectory>false</includeBaseDirectory>
     <includeBaseDirectory>false</includeBaseDirectory>
 
 
     <dependencySets>
     <dependencySets>
         <dependencySet>
         <dependencySet>
             <outputDirectory>lib</outputDirectory>
             <outputDirectory>lib</outputDirectory>
             <useProjectArtifact>false</useProjectArtifact>
             <useProjectArtifact>false</useProjectArtifact>
-            <!--
-            <excludes>
-                <exclude>commons-lang:commons-lang</exclude>
-                <exclude>log4j:log4j</exclude>
-            </excludes>
-            -->
         </dependencySet>
         </dependencySet>
     </dependencySets>
     </dependencySets>
 
 
@@ -40,12 +35,6 @@
                 <include>*</include>
                 <include>*</include>
             </includes>
             </includes>
         </fileSet>
         </fileSet>
-        <fileSet>
-            <directory>${project.build.directory}</directory>
-            <outputDirectory>/lib</outputDirectory>
-            <includes>
-                <include>*.jar</include>
-            </includes>
-        </fileSet>
     </fileSets>
     </fileSets>
+
 </assembly>
 </assembly>

+ 0 - 0
apps/spark/java/sparker-yarn/src/main/bash/run-am.sh → apps/spark/java/livy-yarn/src/main/bash/run-am.sh


+ 0 - 0
apps/spark/java/sparker-yarn/src/main/bash/run-class.sh → apps/spark/java/livy-yarn/src/main/bash/run-class.sh


+ 0 - 0
apps/spark/java/sparker-yarn/src/main/bash/run-job.sh → apps/spark/java/livy-yarn/src/main/bash/run-job.sh


+ 0 - 0
apps/spark/java/sparker-yarn/src/main/resources/log4j.properties → apps/spark/java/livy-yarn/src/main/resources/log4j.properties


+ 16 - 13
apps/spark/java/sparker-yarn/src/main/scala/com/cloudera/hue/sparker/yarn/AppMaster.scala → apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/AppMaster.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.yarn
+package com.cloudera.hue.livy.yarn
 
 
 import org.apache.hadoop.net.NetUtils
 import org.apache.hadoop.net.NetUtils
 import org.apache.hadoop.yarn.api.ApplicationConstants
 import org.apache.hadoop.yarn.api.ApplicationConstants
@@ -28,22 +28,25 @@ object AppMaster extends Logging {
     amRMClient.init(yarnConfig)
     amRMClient.init(yarnConfig)
     amRMClient.start()
     amRMClient.start()
 
 
-    val appMasterHostname = NetUtils.getHostname
-    val appMasterRpcPort = -1
-    val appMasterTrackingUrl = ""
+    try {
+      val appMasterHostname = NetUtils.getHostname
+      val appMasterRpcPort = -1
+      val appMasterTrackingUrl = ""
 
 
-    val response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl)
+      val response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl)
 
 
-    val maxMem = response.getMaximumResourceCapability.getMemory
-    info("max mem capacity on this cluster: %s" format maxMem)
+      val maxMem = response.getMaximumResourceCapability.getMemory
+      info("max mem capacity on this cluster: %s" format maxMem)
 
 
-    val maxVCores = response.getMaximumResourceCapability.getVirtualCores
-    info("max vcore capacity on this cluster: %s" format maxMem)
+      val maxVCores = response.getMaximumResourceCapability.getVirtualCores
+      info("max vcore capacity on this cluster: %s" format maxMem)
+    } finally {
+      val appStatus = FinalApplicationStatus.SUCCEEDED
+      val appMessage = "wee"
 
 
-    val appStatus = FinalApplicationStatus.SUCCEEDED
-    val appMessage = "wee"
-    amRMClient.unregisterApplicationMaster(appStatus, appMessage, null)
-    amRMClient.stop()
+      amRMClient.unregisterApplicationMaster(appStatus, appMessage, null)
+      amRMClient.stop()
+    }
   }
   }
 
 
 }
 }

+ 3 - 14
apps/spark/java/sparker-yarn/src/main/scala/com/cloudera/hue/sparker/yarn/Client.scala → apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/Client.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.yarn
+package com.cloudera.hue.livy.yarn
 
 
 import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.hadoop.yarn.api.ApplicationConstants
 import org.apache.hadoop.yarn.api.ApplicationConstants
@@ -30,17 +30,6 @@ object Client extends Logging {
             ApplicationConstants.LOG_DIR_EXPANSION_VAR,
             ApplicationConstants.LOG_DIR_EXPANSION_VAR,
             ApplicationConstants.LOG_DIR_EXPANSION_VAR
             ApplicationConstants.LOG_DIR_EXPANSION_VAR
           )
           )
-        /*
-          "/bin/pwd " +
-            " 1>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
-            " 2>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr;",
-          "/bin/ls " +
-            " 1>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
-            " 2>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr;",
-          "/bin/echo hi" +
-          " 1>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
-          " 2>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
-          */
         )
         )
       )
       )
 
 
@@ -131,7 +120,7 @@ class Client(yarnConf: YarnConfiguration) {
 
 
     appContext.setApplicationId(appId)
     appContext.setApplicationId(appId)
     appContext.setAMContainerSpec(containerCtx)
     appContext.setAMContainerSpec(containerCtx)
-    appContext.setApplicationType("sparker")
+    appContext.setApplicationType("livy")
 
 
     info("submitting application request for %s" format appId)
     info("submitting application request for %s" format appId)
 
 
@@ -145,7 +134,7 @@ class Client(yarnConf: YarnConfiguration) {
   }
   }
 
 
   private def addToLocalResources(fs: FileSystem, fileSrcPath: String, fileDstPath: String, appId: String): LocalResource = {
   private def addToLocalResources(fs: FileSystem, fileSrcPath: String, fileDstPath: String, appId: String): LocalResource = {
-    val appName = "sparker"
+    val appName = "livy"
     val suffix = appName + "/" + appId + "/" + fileDstPath
     val suffix = appName + "/" + appId + "/" + fileDstPath
 
 
     val dst = new Path(fs.getHomeDirectory, suffix)
     val dst = new Path(fs.getHomeDirectory, suffix)

+ 1 - 1
apps/spark/java/sparker-yarn/src/main/scala/com/cloudera/hue/sparker/yarn/Logging.scala → apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/Logging.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.yarn
+package com.cloudera.hue.livy.yarn
 
 
 import org.slf4j.LoggerFactory
 import org.slf4j.LoggerFactory
 
 

+ 1 - 1
apps/spark/java/sparker-yarn/src/main/scala/com/cloudera/hue/sparker/yarn/YarnJob.scala → apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/YarnJob.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.yarn
+package com.cloudera.hue.livy.yarn
 
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.conf.Configuration
 
 

+ 32 - 16
apps/spark/java/pom.xml

@@ -27,13 +27,13 @@
         <version>3.7.0-SNAPSHOT</version>
         <version>3.7.0-SNAPSHOT</version>
     </parent>
     </parent>
 
 
-    <groupId>com.cloudera.hue.sparker</groupId>
-    <artifactId>sparker-main</artifactId>
+    <groupId>com.cloudera.hue.livy</groupId>
+    <artifactId>livy-main</artifactId>
     <packaging>pom</packaging>
     <packaging>pom</packaging>
     <version>3.7.0-SNAPSHOT</version>
     <version>3.7.0-SNAPSHOT</version>
 
 
-    <name>sparker-main</name>
-    <description>sparker-main</description>
+    <name>livy-main</name>
+    <description>livy-main</description>
 
 
     <licenses>
     <licenses>
         <license>
         <license>
@@ -52,9 +52,12 @@
     </properties>
     </properties>
 
 
     <modules>
     <modules>
-        <module>sparker-repl</module>
-        <module>sparker-server</module>
-        <module>sparker-yarn</module>
+        <!--
+        <module>livy-assembly</module>
+        <module>livy-repl</module>
+        <module>livy-server</module>
+        -->
+        <module>livy-yarn</module>
     </modules>
     </modules>
 
 
     <repositories>
     <repositories>
@@ -213,14 +216,6 @@
                 </configuration>
                 </configuration>
             </plugin>
             </plugin>
 
 
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.4</version>
-                <configuration>
-                    <descriptor>src/main/assembly/dist.xml</descriptor>
-                </configuration>
-            </plugin>
-
             <!--
             <!--
             <plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -228,7 +223,7 @@
                 <configuration>
                 <configuration>
                     <archive>
                     <archive>
                         <manifest>
                         <manifest>
-                            <mainClass>com.cloudera.sparker.SparkerMain</mainClass>
+                            <mainClass>com.cloudera.livy.LivyMain</mainClass>
                         </manifest>
                         </manifest>
                     </archive>
                     </archive>
                 </configuration>
                 </configuration>
@@ -244,6 +239,27 @@
             </plugin>
             </plugin>
             -->
             -->
 
 
+            <!--
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <executions>
+                    <execution>
+                        <id>make-bundles</id>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <phase>package</phase>
+                        <configuration>
+                            <descriptors>
+                                <descriptor>livy-yarn/src/main/assembly/dist.xml</descriptor>
+                            </descriptors>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            -->
+
         </plugins>
         </plugins>
 
 
     </build>
     </build>

+ 0 - 21
apps/spark/java/sparker-repl/src/main/scala/Scalatra.scala

@@ -1,21 +0,0 @@
-import javax.servlet.ServletContext
-
-import _root_.akka.actor.ActorSystem
-import com.cloudera.hue.sparker.repl.interpreter.SparkerInterpreter
-import com.cloudera.hue.sparker.repl.webapp.SparkerApp
-import org.scalatra.LifeCycle
-
-class ScalatraBootstrap extends LifeCycle {
-
-  //val system = ActorSystem()
-  val sparkerInterpreter = new SparkerInterpreter
-
-  override def init(context: ServletContext): Unit = {
-    context.mount(new SparkerApp(sparkerInterpreter), "/*")
-  }
-
-  override def destroy(context: ServletContext): Unit = {
-    sparkerInterpreter.close()
-    //system.shutdown()
-  }
-}

+ 0 - 6
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/SparkerConfiguration.java

@@ -1,6 +0,0 @@
-package com.cloudera.hue.sparker.server;
-
-import io.dropwizard.Configuration;
-
-public class SparkerConfiguration extends Configuration {
-}

+ 0 - 0
apps/spark/java/src/main/assembly/dist.xml → apps/spark/java/src/main/assembly/dist3.xml


+ 0 - 81
apps/spark/spark_server.sh

@@ -1,81 +0,0 @@
-#!/bin/bash
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Runs Beeswax server.
-
-set -o errexit
-
-#if [ -z "$HADOOP_CONF_DIR" ]; then
-#  echo "\$HADOOP_CONF_DIR must be specified" 1>&2
-#  exit 1
-#fi
-#echo \$HADOOP_HOME=$HADOOP_HOME
-#
-#if [ -z "$HADOOP_BIN" ]; then
-#  echo "\$HADOOP_BIN must be specified" 1>&2
-#  exit 1
-#fi
-#echo \$HADOOP_BIN=$HADOOP_BIN
-#
-#if [ -z "$HIVE_CONF_DIR" ]; then
-#  echo "\$HIVE_CONF_DIR must be specified" 1>&2
-#  exit 1
-#fi
-#
-#echo \$HIVE_CONF_DIR=$HIVE_CONF_DIR
-#
-#if [ -z "$HIVE_HOME" ]; then
-#  echo "\$HIVE_HOME not specified. Defaulting to $HIVE_CONF_DIR/.." 1>&2
-#  export HIVE_HOME=$HIVE_CONF_DIR/..
-#  exit 1
-#fi
-#
-#echo \$HIVE_HOME=$HIVE_HOME
-#
-SPARK_ROOT=$(dirname $0)
-SPARK_JAR=$SPARK_ROOT/java/sparker-server/target/sparker-server-3.7.0-SNAPSHOT.jar
-#HIVE_LIB=$HIVE_HOME/lib
-#
-#export HADOOP_CLASSPATH=$(find $HADOOP_HOME -name hue-plugins*.jar | tr "\n" :):$(find $HIVE_LIB -name "*.jar" | tr "\n" :)
-#
-#if [ -n "$HADOOP_EXTRA_CLASSPATH_STRING" ]; then
-#  export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$HADOOP_EXTRA_CLASSPATH_STRING
-#fi
-#
-#export HADOOP_OPTS="-Dlog4j.configuration=log4j.properties"
-#echo \$HADOOP_CLASSPATH=$HADOOP_CLASSPATH
-#echo \$HADOOP_OPTS=$HADOOP_OPTS
-#
-## Use HADOOP_CONF_DIR to preprend to classpath, to avoid fb303 conflict,
-## and to force hive-default to correspond to the Hive version we have.
-## Because we are abusing HADOOP_CONF_DIR, we have to emulate its default
-## behavior here as well.
-#if [ -f $HADOOP_CONF_DIR/hadoop-env.sh ]; then
-#  . $HADOOP_CONF_DIR/hadoop-env.sh
-#fi
-#
-#export HADOOP_CONF_DIR=$HIVE_CONF_DIR:$HADOOP_CONF_DIR
-#echo \$HADOOP_CONF_DIR=$HADOOP_CONF_DIR
-#echo \$HADOOP_MAPRED_HOME=$HADOOP_MAPRED_HOME
-
-export SPARKER_HOME=$(dirname $0)
-
-# Note: I've had trouble running this with just "java -jar" with the classpath
-# determined with a seemingly appropriate find command.
-echo CWD=$(pwd)
-echo Executing java -jar $SPARK_JAR "$@"
-exec java -jar $SPARK_JAR "$@"

+ 0 - 93
apps/spark/sparker-client.py

@@ -1,93 +0,0 @@
-#! /usr/bin/env python
-
-import json
-import httplib
-import urllib
-
-sparker_client_default_host = 'localhost'
-sparker_client_default_port = 8080
-
-class SparkerClient:
-    # Configuration
-    host = sparker_client_default_host
-    port = sparker_client_default_port
-    # State
-    connection = None
-    session_id = None
-    output_cursor = 0
-    # Constants
-    POST = 'POST'
-    GET = 'GET'
-    DELETE = 'DELETE'
-    ROOT = '/'
-    OK = 200
-    def __init__(self, host=sparker_client_default_host, port=sparker_client_default_port, lang=None):
-        self.host = host
-        self.port = port
-        self.connection = self.create_connection()
-        self.session_id = self.create_session(lang)
-    def http_json(self, method, url, body=''):
-        self.connection.request(method, url, body)
-        response = self.connection.getresponse()
-        if response.status != self.OK:
-            raise Exception(str(response.status) + ' ' + response.reason)
-        response_text = response.read()
-        if len(response_text) != 0:
-            return json.loads(response_text)
-        return ''
-    def create_connection(self):
-        return httplib.HTTPConnection(self.host, self.port)
-    def create_session(self, lang):
-        return self.http_json(self.POST, self.ROOT, urllib.urlencode({'lang': lang}))
-    def get_sessions(self):
-        return self.http_json(self.GET, self.ROOT)
-    def get_session(self):
-        return self.http_json(self.GET, self.ROOT + self.session_id)
-    def post_input(self, command):
-        self.http_json(self.POST, self.ROOT + self.session_id, command)
-    def get_output(self):
-        output = self.get_session()[self.output_cursor:]
-        self.output_cursor += len(output)
-        return output
-    def delete_session(self):
-        self.http_json(self.DELETE, self.ROOT + self.session_id)
-    def close_connection(self):
-        self.connection.close()
-
-import threading
-import time
-import sys
-
-class SparkerPoller(threading.Thread):
-    keep_polling = True
-    def __init__(self, sparker_client):
-        threading.Thread.__init__(self)
-        self.sparker_client = sparker_client
-    def stop_polling(self):
-        self.keep_polling = False
-    def run(self):
-        while self.keep_polling:
-            output = self.sparker_client.get_output()
-            for line in output:
-                print(line)
-            time.sleep(1)
-
-if len(sys.argv) == 2:
-    lang = sys.argv[1]
-else:
-    lang = 'scala'
-
-client = SparkerClient(lang=lang)
-poller = SparkerPoller(client)
-poller.start()
-
-try:
-    while True:
-        line = raw_input()
-        client.post_input(line)
-except:
-    poller.stop_polling()
-    client.delete_session()
-    client.close_connection()
-
-sys.exit(0)

+ 0 - 8
apps/spark/sparker-shell

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-cd `dirname $0`
-
-exec java \
-	-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006 \
-	-cp "java/sparker-repl/target/lib/*:java/sparker-repl/target/sparker-repl-3.7.0-SNAPSHOT.jar" \
-	com.cloudera.hue.sparker.repl.Main -usejavacp "$@" 2>/dev/null