Browse Source

[spark] Rename Sparker to Livy since we're going to be more than Spark

Erick Tryzelaar 11 năm trước cách đây
mục cha
commit
9a923e1
34 tập tin đã thay đổi với 288 bổ sung312 xóa
  1. 5 5
      apps/spark/java/livy-repl/pom.xml
  2. 0 0
      apps/spark/java/livy-repl/src/main/assembly/dist.xml
  3. 20 0
      apps/spark/java/livy-repl/src/main/scala/Scalatra.scala
  4. 7 7
      apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/Interpreter.scala
  5. 2 2
      apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/Main.scala
  6. 3 3
      apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/WebApp.scala
  7. 5 5
      apps/spark/java/livy-server/pom.xml
  8. 8 8
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/LivyApp.java
  9. 6 0
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/LivyConfiguration.java
  10. 1 1
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/resources/ExecuteStatementRequest.java
  11. 5 5
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/resources/SessionResource.java
  12. 4 4
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/resources/StatementResource.java
  13. 1 1
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/ClosedSessionException.java
  14. 1 1
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/Session.java
  15. 1 1
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/SessionManager.java
  16. 1 1
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/SparkSession.java
  17. 1 1
      apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/Statement.java
  18. 162 0
      apps/spark/java/livy-yarn/pom.xml
  19. 2 13
      apps/spark/java/livy-yarn/src/main/assembly/dist.xml
  20. 0 0
      apps/spark/java/livy-yarn/src/main/bash/run-am.sh
  21. 0 0
      apps/spark/java/livy-yarn/src/main/bash/run-class.sh
  22. 0 0
      apps/spark/java/livy-yarn/src/main/bash/run-job.sh
  23. 0 0
      apps/spark/java/livy-yarn/src/main/resources/log4j.properties
  24. 16 13
      apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/AppMaster.scala
  25. 3 14
      apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/Client.scala
  26. 1 1
      apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/Logging.scala
  27. 1 1
      apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/YarnJob.scala
  28. 32 16
      apps/spark/java/pom.xml
  29. 0 21
      apps/spark/java/sparker-repl/src/main/scala/Scalatra.scala
  30. 0 6
      apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/SparkerConfiguration.java
  31. 0 0
      apps/spark/java/src/main/assembly/dist3.xml
  32. 0 81
      apps/spark/spark_server.sh
  33. 0 93
      apps/spark/sparker-client.py
  34. 0 8
      apps/spark/sparker-shell

+ 5 - 5
apps/spark/java/sparker-repl/pom.xml → apps/spark/java/livy-repl/pom.xml

@@ -4,13 +4,13 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
-        <groupId>com.cloudera.hue.sparker</groupId>
-        <artifactId>sparker-main</artifactId>
+        <groupId>com.cloudera.hue.livy</groupId>
+        <artifactId>livy-main</artifactId>
         <relativePath>../pom.xml</relativePath>
         <version>3.7.0-SNAPSHOT</version>
     </parent>
 
-    <artifactId>sparker-repl</artifactId>
+    <artifactId>livy-repl</artifactId>
     <packaging>jar</packaging>
 
     <properties>
@@ -124,7 +124,7 @@
                         <manifest>
                             <addClasspath>true</addClasspath>
                             <classpathPrefix>lib/</classpathPrefix>
-                            <mainClass>com.cloudera.hue.sparker.repl.Main</mainClass>
+                            <mainClass>com.cloudera.hue.livy.repl.Main</mainClass>
                         </manifest>
                     </archive>
                 </configuration>
@@ -167,7 +167,7 @@
                             <transformers>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>com.cloudera.hue.sparker.server.SparkerApp</mainClass>
+                                    <mainClass>com.cloudera.hue.livy.server.LivyApp</mainClass>
                                 </transformer>
                             </transformers>
                         </configuration>

+ 0 - 0
apps/spark/java/sparker-repl/src/main/assembly/dist.xml → apps/spark/java/livy-repl/src/main/assembly/dist.xml


+ 20 - 0
apps/spark/java/livy-repl/src/main/scala/Scalatra.scala

@@ -0,0 +1,20 @@
+import javax.servlet.ServletContext
+
+import com.cloudera.hue.livy.repl.interpreter.SparkInterpreter
+import com.cloudera.hue.livy.repl.webapp.LivyApp
+import org.scalatra.LifeCycle
+
+class ScalatraBootstrap extends LifeCycle {
+
+  //val system = ActorSystem()
+  val sparkInterpreter = new SparkInterpreter
+
+  override def init(context: ServletContext): Unit = {
+    context.mount(new LivyApp(sparkInterpreter), "/*")
+  }
+
+  override def destroy(context: ServletContext): Unit = {
+    sparkInterpreter.close()
+    //system.shutdown()
+  }
+}

+ 7 - 7
apps/spark/java/sparker-repl/src/main/scala/com/cloudera/hue/sparker/repl/Interpreter.scala → apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/Interpreter.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.repl.interpreter
+package com.cloudera.hue.livy.repl.interpreter
 
 import java.io.{BufferedReader, PipedReader, PipedWriter, StringWriter}
 import java.util.concurrent.{BlockingQueue, SynchronousQueue}
@@ -10,14 +10,14 @@ import scala.tools.nsc.SparkHelper
 import scala.tools.nsc.interpreter.{Formatting, _}
 import scala.tools.nsc.util.ClassPath
 
-class SparkerInterpreter {
+class SparkInterpreter {
   private implicit def executor: ExecutionContext = ExecutionContext.global
 
   private var running = false;
   private val inQueue = new SynchronousQueue[Request]
   private val inWriter = new PipedWriter()
 
-  org.apache.spark.repl.Main.interp = new SparkerILoop(
+  org.apache.spark.repl.Main.interp = new ILoop(
     this,
     inQueue,
     new BufferedReader(new PipedReader(inWriter)),
@@ -48,13 +48,13 @@ class SparkerInterpreter {
   }
 }
 
-class SparkerILoop(parent: SparkerInterpreter, inQueue: BlockingQueue[Request], in0: BufferedReader, outString: StringWriter) extends SparkILoop(in0, new JPrintWriter(outString)) {
+private class ILoop(parent: SparkInterpreter, inQueue: BlockingQueue[Request], in0: BufferedReader, outString: StringWriter) extends SparkILoop(in0, new JPrintWriter(outString)) {
 
-  class SparkerILoopInterpreter extends SparkILoopInterpreter {
+  class ILoopInterpreter extends SparkILoopInterpreter {
     outer =>
 
     override lazy val formatting = new Formatting {
-      def prompt = SparkerILoop.this.prompt
+      def prompt = ILoop.this.prompt
     }
     override protected def parentClassLoader = SparkHelper.explicitParentLoader(settings).getOrElse(classOf[SparkILoop].getClassLoader)
   }
@@ -69,7 +69,7 @@ class SparkerILoop(parent: SparkerInterpreter, inQueue: BlockingQueue[Request],
       settings.classpath.value)((l, r) => ClassPath.join(l, r))
     this.settings.classpath.value = totalClassPath
 
-    intp = new SparkerILoopInterpreter
+    intp = new ILoopInterpreter
   }
 
   private val replayQuestionMessage =

+ 2 - 2
apps/spark/java/sparker-repl/src/main/scala/com/cloudera/hue/sparker/repl/Main.scala → apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/Main.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.repl
+package com.cloudera.hue.livy.repl
 
 import org.eclipse.jetty.server.Server
 import org.eclipse.jetty.servlet.{ServletHolder, DefaultServlet}
@@ -14,7 +14,7 @@ object Main {
     val context = new WebAppContext()
 
     context.setContextPath("/")
-    context.setResourceBase("src/main/com/cloudera/hue/sparker/repl")
+    context.setResourceBase("src/main/com/cloudera/hue/livy/repl")
     context.addEventListener(new ScalatraListener)
 
     context.addServlet(classOf[DefaultServlet], "/")

+ 3 - 3
apps/spark/java/sparker-repl/src/main/scala/com/cloudera/hue/sparker/repl/WebApp.scala → apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/WebApp.scala

@@ -1,14 +1,14 @@
-package com.cloudera.hue.sparker.repl.webapp
+package com.cloudera.hue.livy.repl.webapp
 
 import akka.util.Timeout
-import com.cloudera.hue.sparker.repl.interpreter.SparkerInterpreter
+import com.cloudera.hue.livy.repl.interpreter.SparkInterpreter
 import org.json4s.{DefaultFormats, Formats}
 import org.scalatra.json._
 import org.scalatra.{Accepted, AsyncResult, FutureSupport, ScalatraServlet}
 
 import scala.concurrent.{Future, ExecutionContext, ExecutionContextExecutor}
 
-class SparkerApp(interpreter: SparkerInterpreter) extends ScalatraServlet with FutureSupport with JacksonJsonSupport {
+class LivyApp(interpreter: SparkInterpreter) extends ScalatraServlet with FutureSupport with JacksonJsonSupport {
 
   protected implicit def executor: ExecutionContextExecutor = ExecutionContext.global
   protected implicit def defaultTimeout: Timeout = Timeout(10)

+ 5 - 5
apps/spark/java/sparker-server/pom.xml → apps/spark/java/livy-server/pom.xml

@@ -21,13 +21,13 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
-        <groupId>com.cloudera.hue.sparker</groupId>
-        <artifactId>sparker-main</artifactId>
+        <groupId>com.cloudera.hue.livy</groupId>
+        <artifactId>livy-main</artifactId>
         <relativePath>../pom.xml</relativePath>
         <version>3.7.0-SNAPSHOT</version>
     </parent>
 
-    <artifactId>sparker-server</artifactId>
+    <artifactId>livy-server</artifactId>
     <packaging>jar</packaging>
 
     <properties>
@@ -124,7 +124,7 @@
                             <transformers>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>com.cloudera.hue.sparker.server.SparkerApp</mainClass>
+                                    <mainClass>com.cloudera.hue.livy.server.LivyApp</mainClass>
                                 </transformer>
                             </transformers>
                         </configuration>
@@ -161,7 +161,7 @@
                         <manifest>
                             <addClasspath>true</addClasspath>
                             <classpathPrefix>lib/</classpathPrefix>
-                            <mainClass>com.cloudera.hue.sparker.server.SparkerMain</mainClass>
+                            <mainClass>com.cloudera.hue.livy.server.LivyMain</mainClass>
                         </manifest>
                     </archive>
                 </configuration>

+ 8 - 8
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/SparkerApp.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/LivyApp.java

@@ -1,8 +1,8 @@
-package com.cloudera.hue.sparker.server;
+package com.cloudera.hue.livy.server;
 
-import com.cloudera.hue.sparker.server.resources.StatementResource;
-import com.cloudera.hue.sparker.server.resources.SessionResource;
-import com.cloudera.hue.sparker.server.sessions.SessionManager;
+import com.cloudera.hue.livy.server.sessions.SessionManager;
+import com.cloudera.hue.livy.server.resources.StatementResource;
+import com.cloudera.hue.livy.server.resources.SessionResource;
 import com.sun.jersey.core.spi.factory.ResponseBuilderImpl;
 import io.dropwizard.Application;
 import io.dropwizard.setup.Bootstrap;
@@ -11,19 +11,19 @@ import io.dropwizard.setup.Environment;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.ext.ExceptionMapper;
 
-public class SparkerApp extends Application<SparkerConfiguration> {
+public class LivyApp extends Application<LivyConfiguration> {
 
     public static void main(String[] args) throws Exception {
-        new SparkerApp().run(args);
+        new LivyApp().run(args);
     }
 
     @Override
-    public void initialize(Bootstrap<SparkerConfiguration> bootstrap) {
+    public void initialize(Bootstrap<LivyConfiguration> bootstrap) {
 
     }
 
     @Override
-    public void run(SparkerConfiguration sparkerConfiguration, Environment environment) throws Exception {
+    public void run(LivyConfiguration livyConfiguration, Environment environment) throws Exception {
         final SessionManager sessionManager = new SessionManager();
         environment.jersey().register(new SessionResource(sessionManager));
         environment.jersey().register(new StatementResource(sessionManager));

+ 6 - 0
apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/LivyConfiguration.java

@@ -0,0 +1,6 @@
+package com.cloudera.hue.livy.server;
+
+import io.dropwizard.Configuration;
+
+public class LivyConfiguration extends Configuration {
+}

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/resources/ExecuteStatementRequest.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/resources/ExecuteStatementRequest.java

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.server.resources;
+package com.cloudera.hue.livy.server.resources;
 
 import org.hibernate.validator.constraints.NotEmpty;
 

+ 5 - 5
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/resources/SessionResource.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/resources/SessionResource.java

@@ -1,9 +1,9 @@
-package com.cloudera.hue.sparker.server.resources;
+package com.cloudera.hue.livy.server.resources;
 
-import com.cloudera.hue.sparker.server.sessions.Statement;
-import com.cloudera.hue.sparker.server.sessions.ClosedSessionException;
-import com.cloudera.hue.sparker.server.sessions.Session;
-import com.cloudera.hue.sparker.server.sessions.SessionManager;
+import com.cloudera.hue.livy.server.sessions.ClosedSessionException;
+import com.cloudera.hue.livy.server.sessions.Session;
+import com.cloudera.hue.livy.server.sessions.SessionManager;
+import com.cloudera.hue.livy.server.sessions.Statement;
 import com.codahale.metrics.annotation.Timed;
 import com.sun.jersey.core.spi.factory.ResponseBuilderImpl;
 

+ 4 - 4
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/resources/StatementResource.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/resources/StatementResource.java

@@ -1,8 +1,8 @@
-package com.cloudera.hue.sparker.server.resources;
+package com.cloudera.hue.livy.server.resources;
 
-import com.cloudera.hue.sparker.server.sessions.Session;
-import com.cloudera.hue.sparker.server.sessions.SessionManager;
-import com.cloudera.hue.sparker.server.sessions.Statement;
+import com.cloudera.hue.livy.server.sessions.Session;
+import com.cloudera.hue.livy.server.sessions.SessionManager;
+import com.cloudera.hue.livy.server.sessions.Statement;
 import com.codahale.metrics.annotation.Timed;
 
 import javax.ws.rs.*;

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/sessions/ClosedSessionException.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/ClosedSessionException.java

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.server.sessions;
+package com.cloudera.hue.livy.server.sessions;
 
 public class ClosedSessionException extends Throwable {
 }

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/sessions/Session.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/Session.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package com.cloudera.hue.sparker.server.sessions;
+package com.cloudera.hue.livy.server.sessions;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/sessions/SessionManager.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/SessionManager.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package com.cloudera.hue.sparker.server.sessions;
+package com.cloudera.hue.livy.server.sessions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/sessions/SparkSession.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/SparkSession.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package com.cloudera.hue.sparker.server.sessions;
+package com.cloudera.hue.livy.server.sessions;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/sessions/Statement.java → apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/Statement.java

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.server.sessions;
+package com.cloudera.hue.livy.server.sessions;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 

+ 162 - 0
apps/spark/java/livy-yarn/pom.xml

@@ -0,0 +1,162 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.cloudera.hue.sparker</groupId>
+        <artifactId>sparker-main</artifactId>
+        <relativePath>../pom.xml</relativePath>
+        <version>3.7.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>sparker-yarn</artifactId>
+    <packaging>jar</packaging>
+
+    <properties>
+        <hadoop.version>2.5.0</hadoop.version>
+        <scala.binary.version>2.10</scala.binary.version>
+        <scala.macros.version>2.0.1</scala.macros.version>
+        <scala.version>2.10.3</scala.version>
+        <PermGen>64m</PermGen>
+        <MaxPermGen>512m</MaxPermGen>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-client</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-api</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+            <version>${scala.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.scala-tools</groupId>
+                <artifactId>maven-scala-plugin</artifactId>
+                <version>2.15.2</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <scalaVersion>${scala.version}</scalaVersion>
+                </configuration
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptor>src/main/assembly/dist.xml</descriptor>
+                </configuration>
+            </plugin>
+
+						<!--
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>1.6</version>
+                <configuration>
+                    <createDependencyReducedPom>true</createDependencyReducedPom>
+                    <filters>
+                        <filter>
+                            <artifact>*:*</artifact>
+                            <excludes>
+                                <exclude>META-INF/*.SF</exclude>
+                                <exclude>META-INF/*.DSA</exclude>
+                                <exclude>META-INF/*.RSA</exclude>
+                            </excludes>
+                        </filter>
+                    </filters>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>com.cloudera.hue.sparker.yarn.Client</mainClass>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+-->
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-dependencies</id>
+                        <phase>prepare-package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
+                            <overWriteReleases>false</overWriteReleases>
+                            <overWriteSnapshots>false</overWriteSnapshots>
+                            <overWriteIfNewer>true</overWriteIfNewer>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <addClasspath>true</addClasspath>
+                            <classpathPrefix>lib/</classpathPrefix>
+                            <maisparkerss>com.cloudera.hue.livy.yarn.Client</mainClass>
+                        </manifest>
+                    </archive>
+                </configuratio>>
+            </plugin>
+
+        </plugins>
+    </build>
+
+    <reporting>
+        <plugins>
+            <plugin>
+                <groupId>org.scala-tools</groupId>
+                <artifactId>maven-scala-plugin</artifactId>
+                <configuration>
+                    <scalaVersion>${scala.version}</scalaVersion>
+                </configuration>
+            </plugin>
+        </plugins>
+    </reporting>
+
+</project>

+ 2 - 13
apps/spark/java/sparker-yarn/src/main/assembly/dist.xml → apps/spark/java/livy-yarn/src/main/assembly/dist.xml

@@ -16,18 +16,13 @@
     <formats>
         <format>tar.gz</format>
     </formats>
+
     <includeBaseDirectory>false</includeBaseDirectory>
 
     <dependencySets>
         <dependencySet>
             <outputDirectory>lib</outputDirectory>
             <useProjectArtifact>false</useProjectArtifact>
-            <!--
-            <excludes>
-                <exclude>commons-lang:commons-lang</exclude>
-                <exclude>log4j:log4j</exclude>
-            </excludes>
-            -->
         </dependencySet>
     </dependencySets>
 
@@ -40,12 +35,6 @@
                 <include>*</include>
             </includes>
         </fileSet>
-        <fileSet>
-            <directory>${project.build.directory}</directory>
-            <outputDirectory>/lib</outputDirectory>
-            <includes>
-                <include>*.jar</include>
-            </includes>
-        </fileSet>
     </fileSets>
+
 </assembly>

+ 0 - 0
apps/spark/java/sparker-yarn/src/main/bash/run-am.sh → apps/spark/java/livy-yarn/src/main/bash/run-am.sh


+ 0 - 0
apps/spark/java/sparker-yarn/src/main/bash/run-class.sh → apps/spark/java/livy-yarn/src/main/bash/run-class.sh


+ 0 - 0
apps/spark/java/sparker-yarn/src/main/bash/run-job.sh → apps/spark/java/livy-yarn/src/main/bash/run-job.sh


+ 0 - 0
apps/spark/java/sparker-yarn/src/main/resources/log4j.properties → apps/spark/java/livy-yarn/src/main/resources/log4j.properties


+ 16 - 13
apps/spark/java/sparker-yarn/src/main/scala/com/cloudera/hue/sparker/yarn/AppMaster.scala → apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/AppMaster.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.yarn
+package com.cloudera.hue.livy.yarn
 
 import org.apache.hadoop.net.NetUtils
 import org.apache.hadoop.yarn.api.ApplicationConstants
@@ -28,22 +28,25 @@ object AppMaster extends Logging {
     amRMClient.init(yarnConfig)
     amRMClient.start()
 
-    val appMasterHostname = NetUtils.getHostname
-    val appMasterRpcPort = -1
-    val appMasterTrackingUrl = ""
+    try {
+      val appMasterHostname = NetUtils.getHostname
+      val appMasterRpcPort = -1
+      val appMasterTrackingUrl = ""
 
-    val response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl)
+      val response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl)
 
-    val maxMem = response.getMaximumResourceCapability.getMemory
-    info("max mem capacity on this cluster: %s" format maxMem)
+      val maxMem = response.getMaximumResourceCapability.getMemory
+      info("max mem capacity on this cluster: %s" format maxMem)
 
-    val maxVCores = response.getMaximumResourceCapability.getVirtualCores
-    info("max vcore capacity on this cluster: %s" format maxMem)
+      val maxVCores = response.getMaximumResourceCapability.getVirtualCores
+      info("max vcore capacity on this cluster: %s" format maxMem)
+    } finally {
+      val appStatus = FinalApplicationStatus.SUCCEEDED
+      val appMessage = "wee"
 
-    val appStatus = FinalApplicationStatus.SUCCEEDED
-    val appMessage = "wee"
-    amRMClient.unregisterApplicationMaster(appStatus, appMessage, null)
-    amRMClient.stop()
+      amRMClient.unregisterApplicationMaster(appStatus, appMessage, null)
+      amRMClient.stop()
+    }
   }
 
 }

+ 3 - 14
apps/spark/java/sparker-yarn/src/main/scala/com/cloudera/hue/sparker/yarn/Client.scala → apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/Client.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.yarn
+package com.cloudera.hue.livy.yarn
 
 import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.hadoop.yarn.api.ApplicationConstants
@@ -30,17 +30,6 @@ object Client extends Logging {
             ApplicationConstants.LOG_DIR_EXPANSION_VAR,
             ApplicationConstants.LOG_DIR_EXPANSION_VAR
           )
-        /*
-          "/bin/pwd " +
-            " 1>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
-            " 2>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr;",
-          "/bin/ls " +
-            " 1>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
-            " 2>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr;",
-          "/bin/echo hi" +
-          " 1>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
-          " 2>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
-          */
         )
       )
 
@@ -131,7 +120,7 @@ class Client(yarnConf: YarnConfiguration) {
 
     appContext.setApplicationId(appId)
     appContext.setAMContainerSpec(containerCtx)
-    appContext.setApplicationType("sparker")
+    appContext.setApplicationType("livy")
 
     info("submitting application request for %s" format appId)
 
@@ -145,7 +134,7 @@ class Client(yarnConf: YarnConfiguration) {
   }
 
   private def addToLocalResources(fs: FileSystem, fileSrcPath: String, fileDstPath: String, appId: String): LocalResource = {
-    val appName = "sparker"
+    val appName = "livy"
     val suffix = appName + "/" + appId + "/" + fileDstPath
 
     val dst = new Path(fs.getHomeDirectory, suffix)

+ 1 - 1
apps/spark/java/sparker-yarn/src/main/scala/com/cloudera/hue/sparker/yarn/Logging.scala → apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/Logging.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.yarn
+package com.cloudera.hue.livy.yarn
 
 import org.slf4j.LoggerFactory
 

+ 1 - 1
apps/spark/java/sparker-yarn/src/main/scala/com/cloudera/hue/sparker/yarn/YarnJob.scala → apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/YarnJob.scala

@@ -1,4 +1,4 @@
-package com.cloudera.hue.sparker.yarn
+package com.cloudera.hue.livy.yarn
 
 import org.apache.hadoop.conf.Configuration
 

+ 32 - 16
apps/spark/java/pom.xml

@@ -27,13 +27,13 @@
         <version>3.7.0-SNAPSHOT</version>
     </parent>
 
-    <groupId>com.cloudera.hue.sparker</groupId>
-    <artifactId>sparker-main</artifactId>
+    <groupId>com.cloudera.hue.livy</groupId>
+    <artifactId>livy-main</artifactId>
     <packaging>pom</packaging>
     <version>3.7.0-SNAPSHOT</version>
 
-    <name>sparker-main</name>
-    <description>sparker-main</description>
+    <name>livy-main</name>
+    <description>livy-main</description>
 
     <licenses>
         <license>
@@ -52,9 +52,12 @@
     </properties>
 
     <modules>
-        <module>sparker-repl</module>
-        <module>sparker-server</module>
-        <module>sparker-yarn</module>
+        <!--
+        <module>livy-assembly</module>
+        <module>livy-repl</module>
+        <module>livy-server</module>
+        -->
+        <module>livy-yarn</module>
     </modules>
 
     <repositories>
@@ -213,14 +216,6 @@
                 </configuration>
             </plugin>
 
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.4</version>
-                <configuration>
-                    <descriptor>src/main/assembly/dist.xml</descriptor>
-                </configuration>
-            </plugin>
-
             <!--
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -228,7 +223,7 @@
                 <configuration>
                     <archive>
                         <manifest>
-                            <mainClass>com.cloudera.sparker.SparkerMain</mainClass>
+                            <mainClass>com.cloudera.livy.LivyMain</mainClass>
                         </manifest>
                     </archive>
                 </configuration>
@@ -244,6 +239,27 @@
             </plugin>
             -->
 
+            <!--
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <executions>
+                    <execution>
+                        <id>make-bundles</id>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <phase>package</phase>
+                        <configuration>
+                            <descriptors>
+                                <descriptor>livy-yarn/src/main/assembly/dist.xml</descriptor>
+                            </descriptors>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            -->
+
         </plugins>
 
     </build>

+ 0 - 21
apps/spark/java/sparker-repl/src/main/scala/Scalatra.scala

@@ -1,21 +0,0 @@
-import javax.servlet.ServletContext
-
-import _root_.akka.actor.ActorSystem
-import com.cloudera.hue.sparker.repl.interpreter.SparkerInterpreter
-import com.cloudera.hue.sparker.repl.webapp.SparkerApp
-import org.scalatra.LifeCycle
-
-class ScalatraBootstrap extends LifeCycle {
-
-  //val system = ActorSystem()
-  val sparkerInterpreter = new SparkerInterpreter
-
-  override def init(context: ServletContext): Unit = {
-    context.mount(new SparkerApp(sparkerInterpreter), "/*")
-  }
-
-  override def destroy(context: ServletContext): Unit = {
-    sparkerInterpreter.close()
-    //system.shutdown()
-  }
-}

+ 0 - 6
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/SparkerConfiguration.java

@@ -1,6 +0,0 @@
-package com.cloudera.hue.sparker.server;
-
-import io.dropwizard.Configuration;
-
-public class SparkerConfiguration extends Configuration {
-}

+ 0 - 0
apps/spark/java/src/main/assembly/dist.xml → apps/spark/java/src/main/assembly/dist3.xml


+ 0 - 81
apps/spark/spark_server.sh

@@ -1,81 +0,0 @@
-#!/bin/bash
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Runs Beeswax server.
-
-set -o errexit
-
-#if [ -z "$HADOOP_CONF_DIR" ]; then
-#  echo "\$HADOOP_CONF_DIR must be specified" 1>&2
-#  exit 1
-#fi
-#echo \$HADOOP_HOME=$HADOOP_HOME
-#
-#if [ -z "$HADOOP_BIN" ]; then
-#  echo "\$HADOOP_BIN must be specified" 1>&2
-#  exit 1
-#fi
-#echo \$HADOOP_BIN=$HADOOP_BIN
-#
-#if [ -z "$HIVE_CONF_DIR" ]; then
-#  echo "\$HIVE_CONF_DIR must be specified" 1>&2
-#  exit 1
-#fi
-#
-#echo \$HIVE_CONF_DIR=$HIVE_CONF_DIR
-#
-#if [ -z "$HIVE_HOME" ]; then
-#  echo "\$HIVE_HOME not specified. Defaulting to $HIVE_CONF_DIR/.." 1>&2
-#  export HIVE_HOME=$HIVE_CONF_DIR/..
-#  exit 1
-#fi
-#
-#echo \$HIVE_HOME=$HIVE_HOME
-#
-SPARK_ROOT=$(dirname $0)
-SPARK_JAR=$SPARK_ROOT/java/sparker-server/target/sparker-server-3.7.0-SNAPSHOT.jar
-#HIVE_LIB=$HIVE_HOME/lib
-#
-#export HADOOP_CLASSPATH=$(find $HADOOP_HOME -name hue-plugins*.jar | tr "\n" :):$(find $HIVE_LIB -name "*.jar" | tr "\n" :)
-#
-#if [ -n "$HADOOP_EXTRA_CLASSPATH_STRING" ]; then
-#  export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$HADOOP_EXTRA_CLASSPATH_STRING
-#fi
-#
-#export HADOOP_OPTS="-Dlog4j.configuration=log4j.properties"
-#echo \$HADOOP_CLASSPATH=$HADOOP_CLASSPATH
-#echo \$HADOOP_OPTS=$HADOOP_OPTS
-#
-## Use HADOOP_CONF_DIR to preprend to classpath, to avoid fb303 conflict,
-## and to force hive-default to correspond to the Hive version we have.
-## Because we are abusing HADOOP_CONF_DIR, we have to emulate its default
-## behavior here as well.
-#if [ -f $HADOOP_CONF_DIR/hadoop-env.sh ]; then
-#  . $HADOOP_CONF_DIR/hadoop-env.sh
-#fi
-#
-#export HADOOP_CONF_DIR=$HIVE_CONF_DIR:$HADOOP_CONF_DIR
-#echo \$HADOOP_CONF_DIR=$HADOOP_CONF_DIR
-#echo \$HADOOP_MAPRED_HOME=$HADOOP_MAPRED_HOME
-
-export SPARKER_HOME=$(dirname $0)
-
-# Note: I've had trouble running this with just "java -jar" with the classpath
-# determined with a seemingly appropriate find command.
-echo CWD=$(pwd)
-echo Executing java -jar $SPARK_JAR "$@"
-exec java -jar $SPARK_JAR "$@"

+ 0 - 93
apps/spark/sparker-client.py

@@ -1,93 +0,0 @@
-#! /usr/bin/env python
-
-import json
-import httplib
-import urllib
-
-sparker_client_default_host = 'localhost'
-sparker_client_default_port = 8080
-
-class SparkerClient:
-    # Configuration
-    host = sparker_client_default_host
-    port = sparker_client_default_port
-    # State
-    connection = None
-    session_id = None
-    output_cursor = 0
-    # Constants
-    POST = 'POST'
-    GET = 'GET'
-    DELETE = 'DELETE'
-    ROOT = '/'
-    OK = 200
-    def __init__(self, host=sparker_client_default_host, port=sparker_client_default_port, lang=None):
-        self.host = host
-        self.port = port
-        self.connection = self.create_connection()
-        self.session_id = self.create_session(lang)
-    def http_json(self, method, url, body=''):
-        self.connection.request(method, url, body)
-        response = self.connection.getresponse()
-        if response.status != self.OK:
-            raise Exception(str(response.status) + ' ' + response.reason)
-        response_text = response.read()
-        if len(response_text) != 0:
-            return json.loads(response_text)
-        return ''
-    def create_connection(self):
-        return httplib.HTTPConnection(self.host, self.port)
-    def create_session(self, lang):
-        return self.http_json(self.POST, self.ROOT, urllib.urlencode({'lang': lang}))
-    def get_sessions(self):
-        return self.http_json(self.GET, self.ROOT)
-    def get_session(self):
-        return self.http_json(self.GET, self.ROOT + self.session_id)
-    def post_input(self, command):
-        self.http_json(self.POST, self.ROOT + self.session_id, command)
-    def get_output(self):
-        output = self.get_session()[self.output_cursor:]
-        self.output_cursor += len(output)
-        return output
-    def delete_session(self):
-        self.http_json(self.DELETE, self.ROOT + self.session_id)
-    def close_connection(self):
-        self.connection.close()
-
-import threading
-import time
-import sys
-
-class SparkerPoller(threading.Thread):
-    keep_polling = True
-    def __init__(self, sparker_client):
-        threading.Thread.__init__(self)
-        self.sparker_client = sparker_client
-    def stop_polling(self):
-        self.keep_polling = False
-    def run(self):
-        while self.keep_polling:
-            output = self.sparker_client.get_output()
-            for line in output:
-                print(line)
-            time.sleep(1)
-
-if len(sys.argv) == 2:
-    lang = sys.argv[1]
-else:
-    lang = 'scala'
-
-client = SparkerClient(lang=lang)
-poller = SparkerPoller(client)
-poller.start()
-
-try:
-    while True:
-        line = raw_input()
-        client.post_input(line)
-except:
-    poller.stop_polling()
-    client.delete_session()
-    client.close_connection()
-
-sys.exit(0)

+ 0 - 8
apps/spark/sparker-shell

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-cd `dirname $0`
-
-exec java \
-	-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006 \
-	-cp "java/sparker-repl/target/lib/*:java/sparker-repl/target/sparker-repl-3.7.0-SNAPSHOT.jar" \
-	com.cloudera.hue.sparker.repl.Main -usejavacp "$@" 2>/dev/null