浏览代码

[spark] Get sparker-shell integerated into SparkServer

Erick Tryzelaar 11 年之前
父节点
当前提交
5ce5958d35

+ 2 - 1
apps/spark/java/sparker-repl/src/main/scala/com/cloudera/hue/sparker/repl/SparkerILoop.scala

@@ -83,7 +83,8 @@ class SparkerILoop(in0: BufferedReader, outString: StringWriter) extends SparkIL
       else command(line) match {
         case Result(false, _)           => false
         case Result(_, Some(finalLine)) => {
-          val output: String = outString.getBuffer.toString.substring("scala> ".length)
+          var output: String = outString.getBuffer.toString
+          output = output.substring("scala> ".length + 1, output.length - 1)
           outString.getBuffer.setLength(0)
           println(compact(render(Map("type" -> "result", "input" -> finalLine, "output" -> output))))
           addReplay(finalLine)

+ 19 - 4
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/SparkerSession.java

@@ -19,6 +19,8 @@
 package com.cloudera.hue.sparker.server;
 
 import com.google.common.collect.Lists;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -42,10 +44,11 @@ public class SparkerSession implements Session {
 
         this.key = key;
 
-        ProcessBuilder pb = new ProcessBuilder("spark-shell")
+        String sparker_home = System.getenv("SPARKER_HOME");
+
+        ProcessBuilder pb = new ProcessBuilder(Lists.newArrayList(sparker_home + "/sparker-shell"))
                 .redirectInput(ProcessBuilder.Redirect.PIPE)
-                .redirectOutput(ProcessBuilder.Redirect.PIPE)
-                .redirectErrorStream(true);
+                .redirectOutput(ProcessBuilder.Redirect.PIPE);
 
         this.process = pb.start();
 
@@ -57,8 +60,20 @@ public class SparkerSession implements Session {
                 try {
                     String line;
 
+                    ObjectMapper mapper = new ObjectMapper();
+
                     while ((line = reader.readLine()) != null) {
-                        outputLines.add(line);
+                        JsonNode node = mapper.readTree(line);
+                        String type = node.get("type").asText();
+                        if (type.equals("ready")) {
+                            outputLines.add("> ");
+                        } else if (type.equals("done")) {
+                            break;
+                        } else  if (type.equals("result")) {
+                            String output = node.get("output").asText();
+                            outputLines.add(output);
+                            outputLines.add("> ");
+                        }
                     }
 
                     process.waitFor();

+ 4 - 2
apps/spark/spark_server.sh

@@ -72,8 +72,10 @@ SPARK_JAR=$SPARK_ROOT/java/sparker-server/target/sparker-server-3.7.0-SNAPSHOT.j
 #echo \$HADOOP_CONF_DIR=$HADOOP_CONF_DIR
 #echo \$HADOOP_MAPRED_HOME=$HADOOP_MAPRED_HOME
 
+export SPARKER_HOME=$(dirname $0)
+
 # Note: I've had trouble running this with just "java -jar" with the classpath
 # determined with a seemingly appropriate find command.
 echo CWD=$(pwd)
-echo Executing java jar $SPARK_JAR "$@"
-exec java -jar $SPARK_JAR "$@"
+echo Executing java -jar $SPARK_JAR "$@"
+exec java -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005 -jar $SPARK_JAR "$@"

+ 1 - 1
apps/spark/java/sparker-client.py → apps/spark/sparker-client.py

@@ -29,7 +29,7 @@ class SparkerClient:
         self.connection.request(method, url, body)
         response = self.connection.getresponse()
         if response.status != self.OK:
-            raise Exception(str(resonse.status) + ' ' + response.reason)
+            raise Exception(str(response.status) + ' ' + response.reason)
         response_text = response.read()
         if len(response_text) != 0:
             return json.loads(response_text)

+ 3 - 0
apps/spark/sparker-shell

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec java -cp "java/sparker-repl/target/lib/*:java/sparker-repl/target/sparker-repl-3.7.0-SNAPSHOT.jar" com.cloudera.hue.sparker.repl.Main -usejavacp "$@"