|
|
@@ -1,8 +1,10 @@
|
|
|
package com.cloudera.hue.livy.server.batch
|
|
|
|
|
|
+import java.lang.ProcessBuilder.Redirect
|
|
|
import java.util.concurrent.ConcurrentHashMap
|
|
|
import java.util.concurrent.atomic.AtomicInteger
|
|
|
|
|
|
+import com.cloudera.hue.livy.LineBufferedProcess
|
|
|
import com.cloudera.hue.livy.spark.SparkSubmitProcessBuilder
|
|
|
|
|
|
import scala.collection.JavaConversions._
|
|
|
@@ -56,17 +58,45 @@ abstract class BatchFactory {
|
|
|
|
|
|
class BatchProcessFactory extends BatchFactory {
|
|
|
def createBatch(id: Int, createBatchRequest: CreateBatchRequest): Batch =
|
|
|
- BatchProcess(id, createBatchRequest)
|
|
|
+ BatchProcess(id, "local[*]", createBatchRequest)
|
|
|
+}
|
|
|
+
|
|
|
+class BatchYarnFactory extends BatchFactory {
|
|
|
+ def createBatch(id: Int, createBatchRequest: CreateBatchRequest): Batch =
|
|
|
+ BatchProcess(id, "yarn-client", createBatchRequest)
|
|
|
+}
|
|
|
+
|
|
|
+sealed trait State
|
|
|
+
|
|
|
+case class Running() extends State {
|
|
|
+ override def toString = "running"
|
|
|
+}
|
|
|
+
|
|
|
+case class Dead() extends State {
|
|
|
+ override def toString = "dead"
|
|
|
}
|
|
|
|
|
|
abstract class Batch {
|
|
|
def id: Int
|
|
|
|
|
|
+ def state: State
|
|
|
+
|
|
|
+ def lines: IndexedSeq[String]
|
|
|
+
|
|
|
def stop(): Future[Unit]
|
|
|
}
|
|
|
|
|
|
object BatchProcess {
|
|
|
- def apply(id: Int, createBatchRequest: CreateBatchRequest): Batch = {
|
|
|
+ def apply(id: Int, master: String, createBatchRequest: CreateBatchRequest): Batch = {
|
|
|
+ val builder = sparkBuilder(createBatchRequest)
|
|
|
+
|
|
|
+ builder.master(master)
|
|
|
+
|
|
|
+ val process = builder.start(createBatchRequest.file, createBatchRequest.args)
|
|
|
+ new BatchProcess(id, new LineBufferedProcess(process))
|
|
|
+ }
|
|
|
+
|
|
|
+ private def sparkBuilder(createBatchRequest: CreateBatchRequest): SparkSubmitProcessBuilder = {
|
|
|
val builder = SparkSubmitProcessBuilder()
|
|
|
|
|
|
createBatchRequest.className.foreach(builder.className)
|
|
|
@@ -79,20 +109,43 @@ object BatchProcess {
|
|
|
createBatchRequest.executorCores.foreach(builder.executorCores)
|
|
|
createBatchRequest.archives.foreach(builder.archive)
|
|
|
|
|
|
- val process = builder.start(createBatchRequest.file, createBatchRequest.args)
|
|
|
- new BatchProcess(id, process)
|
|
|
+ builder.redirectOutput(Redirect.PIPE)
|
|
|
+
|
|
|
+ builder
|
|
|
}
|
|
|
}
|
|
|
|
|
|
private class BatchProcess(val id: Int,
|
|
|
- @transient
|
|
|
- process: Process) extends Batch {
|
|
|
+ process: LineBufferedProcess) extends Batch {
|
|
|
protected implicit def executor: ExecutionContextExecutor = ExecutionContext.global
|
|
|
|
|
|
+ private[this] var isAlive = true
|
|
|
+
|
|
|
+ override def state: State = {
|
|
|
+ if (isAlive) {
|
|
|
+ try {
|
|
|
+ process.exitValue()
|
|
|
+ } catch {
|
|
|
+ case e: IllegalThreadStateException => return Running()
|
|
|
+ }
|
|
|
+
|
|
|
+ destroyProcess()
|
|
|
+ }
|
|
|
+
|
|
|
+ Dead()
|
|
|
+ }
|
|
|
+
|
|
|
+ override def lines: IndexedSeq[String] = process.stdoutLines
|
|
|
+
|
|
|
override def stop(): Future[Unit] = {
|
|
|
Future {
|
|
|
- process.destroy()
|
|
|
- process.waitFor()
|
|
|
+ destroyProcess()
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
+ private def destroyProcess() = {
|
|
|
+ process.destroy()
|
|
|
+ process.waitFor()
|
|
|
+ isAlive = false
|
|
|
+ }
|
|
|
}
|