浏览代码

[spark] Bump garbage collecting sessions to an hour

Erick Tryzelaar 11 年之前
父节点
当前提交
12ec268

+ 29 - 2
apps/spark/java/sparker-repl/pom.xml

@@ -29,7 +29,24 @@
             <groupId>org.apache.spark</groupId>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-repl_2.10</artifactId>
             <artifactId>spark-repl_2.10</artifactId>
             <version>${spark.version}</version>
             <version>${spark.version}</version>
+
+            <!--
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-beanutils</groupId>
+                    <artifactId>commons-beanutils</artifactId>
+                </exclusion>
+            </exclusions>
+            -->
+        </dependency>
+
+        <!--
+        <dependency>
+            <groupId>commons-beanutils</groupId>
+            <artifactId>commons-beanutils</artifactId>
+            <version>1.8.0</version>
         </dependency>
         </dependency>
+        -->
 
 
         <dependency>
         <dependency>
             <groupId>org.json4s</groupId>
             <groupId>org.json4s</groupId>
@@ -42,20 +59,31 @@
             <artifactId>scalatra_2.10</artifactId>
             <artifactId>scalatra_2.10</artifactId>
             <version>${scalatra.version}</version>
             <version>${scalatra.version}</version>
             <scope>compile</scope>
             <scope>compile</scope>
+            <!--
+            <exclusions>
+                <exclusion>
+                    <groupId>com.typesafe.akka</groupId>
+                    <artifactId>akka-actor_2.10</artifactId>
+                </exclusion>
+            </exclusions>
+            -->
         </dependency>
         </dependency>
 
 
+        <!--
         <dependency>
         <dependency>
             <groupId>org.scalatra</groupId>
             <groupId>org.scalatra</groupId>
             <artifactId>scalatra_2.10</artifactId>
             <artifactId>scalatra_2.10</artifactId>
             <version>${scalatra.version}</version>
             <version>${scalatra.version}</version>
             <scope>compile</scope>
             <scope>compile</scope>
         </dependency>
         </dependency>
+        -->
 
 
     </dependencies>
     </dependencies>
 
 
     <build>
     <build>
         <plugins>
         <plugins>
 
 
+            <!--
             <plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
                 <artifactId>maven-shade-plugin</artifactId>
@@ -90,6 +118,7 @@
                     </execution>
                     </execution>
                 </executions>
                 </executions>
             </plugin>
             </plugin>
+            -->
 
 
             <plugin>
             <plugin>
                 <groupId>org.eclipse.jetty</groupId>
                 <groupId>org.eclipse.jetty</groupId>
@@ -121,7 +150,6 @@
                 </configuration>
                 </configuration>
             </plugin>
             </plugin>
 
 
-            <!--
             <plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-dependency-plugin</artifactId>
                 <artifactId>maven-dependency-plugin</artifactId>
@@ -154,7 +182,6 @@
                     </archive>
                     </archive>
                 </configuration>
                 </configuration>
             </plugin>
             </plugin>
-            -->
         </plugins>
         </plugins>
     </build>
     </build>
 
 

+ 9 - 26
apps/spark/java/sparker-repl/src/main/scala/Scalatra.scala

@@ -1,36 +1,19 @@
+import java.io.StringWriter
 import javax.servlet.ServletContext
 import javax.servlet.ServletContext
 
 
-import org.eclipse.jetty.server.Server
-import org.eclipse.jetty.servlet.DefaultServlet
-import org.eclipse.jetty.webapp.WebAppContext
-import org.scalatra.{ScalatraFilter, LifeCycle}
-import org.scalatra.servlet.ScalatraListener
+import com.cloudera.hue.sparker.repl.{HelloWorldApp, SparkerILoop}
+import org.scalatra.LifeCycle
 
 
-class HelloWorldApp extends ScalatraFilter {
-  get("/") {
-    <h1>Hello {params("name")}</h1>
+trait SparkerILoopInit {
+  def configureSparkerILoop() {
+    org.apache.spark.repl.Main.interp = new SparkerILoop(Console.in, new StringWriter)
+    org.apache.spark.repl.Main.interp.process(new Array[String](0))
   }
   }
 }
 }
 
 
-object Main {
-  def main(args: Array[String]): Unit = {
-    val port = 8087
-    val server = new Server(port)
-    val context = new WebAppContext()
-    context.setContextPath("/")
-    context.setResourceBase("src/main/com/cloudera/hue/sparker/repl")
-    context.addEventListener(new ScalatraListener)
-    context.addServlet(classOf[DefaultServlet], "/")
-
-    server.setHandler(context)
-
-    server.start()
-    server.join()
-  }
-}
-
-class ScalatraBootstrap extends LifeCycle {
+class ScalatraBootstrap extends LifeCycle with SparkerILoopInit {
   override def init(context: ServletContext): Unit = {
   override def init(context: ServletContext): Unit = {
+    configureSparkerILoop()
     context.mount(new HelloWorldApp, "/*")
     context.mount(new HelloWorldApp, "/*")
   }
   }
 }
 }

+ 9 - 0
apps/spark/java/sparker-repl/src/main/scala/com/cloudera/hue/sparker/repl/HelloWorldApp.scala

@@ -0,0 +1,9 @@
+package com.cloudera.hue.sparker.repl
+
+import org.scalatra.ScalatraFilter
+
+class HelloWorldApp extends ScalatraFilter {
+  get("/") {
+    <h1>Hello {params("name")}</h1>
+  }
+}

+ 24 - 0
apps/spark/java/sparker-repl/src/main/scala/com/cloudera/hue/sparker/repl/Main.scala

@@ -2,9 +2,33 @@ package com.cloudera.hue.sparker.repl
 
 
 import java.io.StringWriter
 import java.io.StringWriter
 
 
+import org.eclipse.jetty.server.Server
+import org.eclipse.jetty.servlet.DefaultServlet
+import org.eclipse.jetty.webapp.WebAppContext
+import org.scalatra.servlet.ScalatraListener
+
 object Main {
 object Main {
   def main(args: Array[String]): Unit = {
   def main(args: Array[String]): Unit = {
     org.apache.spark.repl.Main.interp = new SparkerILoop(Console.in, new StringWriter)
     org.apache.spark.repl.Main.interp = new SparkerILoop(Console.in, new StringWriter)
     org.apache.spark.repl.Main.interp.process(args)
     org.apache.spark.repl.Main.interp.process(args)
   }
   }
 }
 }
+
+/*
+object Main {
+  def main(args: Array[String]): Unit = {
+    val port = 8087
+    val server = new Server(port)
+    val context = new WebAppContext()
+    context.setContextPath("/")
+    context.setResourceBase("src/main/com/cloudera/hue/sparker/repl")
+    context.addEventListener(new ScalatraListener)
+    context.addServlet(classOf[DefaultServlet], "/")
+
+    server.setHandler(context)
+
+    server.start()
+    server.join()
+  }
+}
+*/

+ 1 - 1
apps/spark/java/sparker-server/src/main/java/com/cloudera/hue/sparker/server/sessions/SessionManager.java

@@ -112,7 +112,7 @@ public class SessionManager {
 
 
     private class SessionManagerGarbageCollector extends Thread {
     private class SessionManagerGarbageCollector extends Thread {
 
 
-        protected long period = 600000; // Time in milliseconds; TODO: make configurable
+        protected long period = 1000 * 60 * 60; // Time in milliseconds; TODO: make configurable
 
 
         public SessionManagerGarbageCollector() {
         public SessionManagerGarbageCollector() {
             super();
             super();

+ 2 - 3
apps/spark/sparker-shell

@@ -2,8 +2,7 @@
 
 
 cd `dirname $0`
 cd `dirname $0`
 
 
-	#-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006 \
-
 exec java \
 exec java \
+	-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006 \
 	-cp "java/sparker-repl/target/lib/*:java/sparker-repl/target/sparker-repl-3.7.0-SNAPSHOT.jar" \
 	-cp "java/sparker-repl/target/lib/*:java/sparker-repl/target/sparker-repl-3.7.0-SNAPSHOT.jar" \
-	com.cloudera.hue.sparker.repl.Main -usejavacp "$@" 2>/dev/null
+	com.cloudera.hue.sparker.repl.Main #-usejavacp "$@" 2>/dev/null