|
|
@@ -1,23 +1,18 @@
|
|
|
package com.cloudera.hue.livy.server
|
|
|
|
|
|
-import java.util.concurrent.TimeoutException
|
|
|
-
|
|
|
-import com.cloudera.hue.livy.{ExecuteRequest, ExecuteResponse, Logging}
|
|
|
-import dispatch._, Defaults._
|
|
|
-import org.json4s.JsonDSL._
|
|
|
-import org.json4s.jackson.JsonMethods._
|
|
|
-import org.json4s.jackson.Serialization.write
|
|
|
-import org.json4s.{DefaultFormats, Formats}
|
|
|
-
|
|
|
import scala.annotation.tailrec
|
|
|
-import scala.concurrent.duration._
|
|
|
-import scala.concurrent.{Await, ExecutionContext, ExecutionContextExecutor, Future}
|
|
|
+import scala.concurrent.Future
|
|
|
import scala.io.Source
|
|
|
|
|
|
object SparkProcessSession {
|
|
|
val LIVY_HOME = System.getenv("LIVY_HOME")
|
|
|
val SPARK_SHELL = LIVY_HOME + "/spark-shell"
|
|
|
|
|
|
+ def create(id: String): Session = {
|
|
|
+ val (process, port) = startProcess()
|
|
|
+ new SparkProcessSession(id, process, port)
|
|
|
+ }
|
|
|
+
|
|
|
// Loop until we've started a process with a valid port.
|
|
|
private def startProcess(): (Process, Int) = {
|
|
|
val regex = """Starting livy-repl on port (\d+)""".r
|
|
|
@@ -60,103 +55,14 @@ object SparkProcessSession {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-class SparkProcessSession(val id: String) extends Session with Logging {
|
|
|
-
|
|
|
- import com.cloudera.hue.livy.server.SparkProcessSession._
|
|
|
-
|
|
|
- private[this] implicit def executor: ExecutionContextExecutor = ExecutionContext.global
|
|
|
- private[this] implicit def jsonFormats: Formats = DefaultFormats
|
|
|
-
|
|
|
- private[this] var _lastActivity = Long.MaxValue
|
|
|
- private[this] var _state: State = Running()
|
|
|
- private[this] val (process, port) = startProcess()
|
|
|
- private[this] val svc = host("localhost", port)
|
|
|
-
|
|
|
- override def lastActivity: Long = _lastActivity
|
|
|
-
|
|
|
- override def state: State = _state
|
|
|
-
|
|
|
- override def executeStatement(statement: String): Future[ExecuteResponse] = {
|
|
|
- ensureRunning {
|
|
|
- touchLastActivity()
|
|
|
-
|
|
|
- var req = (svc / "statements").setContentType("application/json", "UTF-8")
|
|
|
- req = req << write(ExecuteRequest(statement))
|
|
|
-
|
|
|
- for {
|
|
|
- body <- Http(req OK as.json4s.Json)
|
|
|
- } yield body.extract[ExecuteResponse]
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- override def statement(statementId: Int): Future[ExecuteResponse] = {
|
|
|
- ensureRunning {
|
|
|
- val req = svc / "statements" / statementId
|
|
|
-
|
|
|
- for {
|
|
|
- body <- Http(req OK as.json4s.Json)
|
|
|
- } yield body.extract[ExecuteResponse]
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- override def statements(): Future[List[ExecuteResponse]] = {
|
|
|
- ensureRunning {
|
|
|
- val req = svc / "statements"
|
|
|
-
|
|
|
- for {
|
|
|
- body <- Http(req OK as.json4s.Json)
|
|
|
- } yield body.extract[List[ExecuteResponse]]
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- override def statements(fromIndex: Integer, toIndex: Integer): Future[List[ExecuteResponse]] = {
|
|
|
- ensureRunning {
|
|
|
- val req = (svc / "statements")
|
|
|
- .addQueryParameter("from", fromIndex.toString)
|
|
|
- .addQueryParameter("to", toIndex.toString)
|
|
|
-
|
|
|
- for {
|
|
|
- body <- Http(req OK as.json4s.Json)
|
|
|
- } yield body.extract[List[ExecuteResponse]]
|
|
|
- }
|
|
|
- }
|
|
|
- override def interrupt(): Unit = {
|
|
|
- close()
|
|
|
- }
|
|
|
-
|
|
|
- override def close(): Unit = {
|
|
|
- synchronized {
|
|
|
- _state match {
|
|
|
- case Running() =>
|
|
|
- _state = Stopping()
|
|
|
-
|
|
|
- // Give the repl some time to shut down cleanly.
|
|
|
- try {
|
|
|
- Await.ready(Http(svc.DELETE OK as.String), 5 seconds)
|
|
|
- } catch {
|
|
|
- // Ignore timeouts
|
|
|
- case _: TimeoutException =>
|
|
|
- case _: InterruptedException =>
|
|
|
- }
|
|
|
+private class SparkProcessSession(id: String, process: Process, port: Int) extends SparkWebSession(id, "localhost", port) {
|
|
|
|
|
|
- process.destroy()
|
|
|
- _state = Stopped()
|
|
|
- case Stopping() | Stopped() =>
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
+ override def close(): Future[Unit] = {
|
|
|
+ super.close() andThen { case r =>
|
|
|
+ // Make sure the process is reaped.
|
|
|
+ process.waitFor()
|
|
|
|
|
|
- private def touchLastActivity() = {
|
|
|
- _lastActivity = System.currentTimeMillis()
|
|
|
- }
|
|
|
-
|
|
|
- private def ensureRunning[A](f: => A) = {
|
|
|
- synchronized {
|
|
|
- if (_state == Running()) {
|
|
|
- f
|
|
|
- } else {
|
|
|
- throw new IllegalStateException("Session is in state %s" format _state)
|
|
|
- }
|
|
|
+ r
|
|
|
}
|
|
|
}
|
|
|
}
|