ソースを参照

Reload yarn-site.conf to get resource manager address in case get application status failed.

Tianjin Gu 10 年 前
コミット
02f4e57

+ 1 - 0
apps/spark/java/livy-server/src/main/scala/com/cloudera/hue/livy/server/batch/BatchSessionProcess.scala

@@ -46,6 +46,7 @@ object BatchSessionProcess {
     createBatchRequest.driverCores.foreach(builder.driverCores)
     createBatchRequest.executorMemory.foreach(builder.executorMemory)
     createBatchRequest.executorCores.foreach(builder.executorCores)
+    createBatchRequest.numExecutors.foreach(builder.numExecutors)
     createBatchRequest.archives.map(RelativePath).foreach(builder.archive)
     createBatchRequest.proxyUser.foreach(builder.proxyUser)
 

+ 1 - 1
apps/spark/java/livy-server/src/main/scala/com/cloudera/hue/livy/server/batch/BatchSessionYarn.scala

@@ -21,7 +21,7 @@ package com.cloudera.hue.livy.server.batch
 import java.lang.ProcessBuilder.Redirect
 
 import com.cloudera.hue.livy.sessions._
-import com.cloudera.hue.livy.spark.SparkSubmitProcessBuilder.RelativePath
+import com.cloudera.hue.livy.spark.SparkSubmitProcessBuilder.{RelativePath,  AbsolutePath}
 import com.cloudera.hue.livy.{LineBufferedProcess, LivyConf}
 import com.cloudera.hue.livy.spark.SparkSubmitProcessBuilder
 import com.cloudera.hue.livy.yarn._

+ 5 - 0
apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/Client.scala

@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.{ApplicationId, FinalApplicationStatus
 import org.apache.hadoop.yarn.client.api.YarnClient
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.hadoop.yarn.util.ConverterUtils
+import org.apache.hadoop.fs.Path
 
 import scala.annotation.tailrec
 import scala.concurrent.ExecutionContext
@@ -51,6 +52,10 @@ class Client(livyConf: LivyConf) extends Logging {
 
   private[this] val yarnConf = new YarnConfiguration()
   private[this] val yarnClient = YarnClient.createYarnClient()
+  val path = new Path(sys.env("HADOOP_CONF_DIR") + YarnConfiguration.YARN_SITE_CONFIGURATION_FILE)
+  yarnConf.addResource(path)
+  val rm_address = yarnConf.get(YarnConfiguration.RM_ADDRESS)
+  info(s"Resource Manager address: $rm_address")
   yarnClient.init(yarnConf)
   yarnClient.start()