Browse Source

[livy] Fix yarn sessions

Erick Tryzelaar 10 years ago
parent
commit
bf7f210

+ 3 - 1
apps/spark/java/bin/livy-repl

@@ -21,6 +21,8 @@ set -e
 
 
 export LIVY_HOME=$(cd $(dirname $0)/.. && pwd)
 export LIVY_HOME=$(cd $(dirname $0)/.. && pwd)
 
 
+CLASSPATH="$LIVY_HOME/livy-repl/target/lib/*:$LIVY_HOME/livy-repl/target/livy-repl-3.7.0-SNAPSHOT.jar:$CLASSPATH"
+
 exec java \
 exec java \
-	-cp "$LIVY_HOME/livy-repl/target/lib/*:$LIVY_HOME/livy-repl/target/livy-repl-3.7.0-SNAPSHOT.jar" \
+	-cp "$CLASSPATH" \
 	com.cloudera.hue.livy.repl.Main "$@"
 	com.cloudera.hue.livy.repl.Main "$@"

+ 3 - 1
apps/spark/java/bin/livy-server

@@ -21,7 +21,9 @@ set -e
 
 
 export LIVY_HOME=$(cd $(dirname $0)/.. && pwd)
 export LIVY_HOME=$(cd $(dirname $0)/.. && pwd)
 
 
+CLASSPATH="$LIVY_HOME/livy-server/target/lib/*:$LIVY_HOME/livy-server/target/livy-server-3.7.0-SNAPSHOT.jar:$CLASSPATH"
+
 exec java \
 exec java \
 	-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 \
 	-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 \
-	-cp "$LIVY_HOME/livy-server/target/lib/*:$LIVY_HOME/livy-server/target/livy-server-3.7.0-SNAPSHOT.jar" \
+	-cp "$CLASSPATH" \
 	com.cloudera.hue.livy.server.Main "$@"
 	com.cloudera.hue.livy.server.Main "$@"

+ 28 - 0
apps/spark/java/bin/livy-yarn-client

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Runs Livy server.
+
+set -e
+
+export LIVY_HOME=$(cd $(dirname $0)/.. && pwd)
+
+CLASSPATH="$LIVY_HOME/livy-server/target/lib/*:$LIVY_HOME/livy-server/target/livy-server-3.7.0-SNAPSHOT.jar:`hadoop classpath`:$CLASSPATH"
+
+exec java \
+	-cp "$CLASSPATH" \
+	com.cloudera.hue.livy.yarn.Client "$@"

+ 1 - 1
apps/spark/java/livy-server/src/main/scala/com/cloudera/hue/livy/server/sessions/YarnSession.scala

@@ -26,7 +26,7 @@ object YarnSession {
     )
     )
 
 
     Future {
     Future {
-      var x = job.waitForRPC(10000)
+      var x = job.waitForRPC(100000)
 
 
       println("x: %s" format x)
       println("x: %s" format x)
 
 

+ 0 - 2
apps/spark/java/livy-yarn/src/main/bash/run-class.sh

@@ -36,7 +36,5 @@ function check_and_enable_64_bit_mode {
 # Check if logback configuration is specified. If not - set to resources/logback.xml
 # Check if logback configuration is specified. If not - set to resources/logback.xml
 [[ $JAVA_OPTS != *-Dlogback.configuration* && -f $DEFAULT_LOGBACK_FILE ]] && JAVA_OPTS="$JAVA_OPTS -Dlogback.configuration=file:$DEFAULT_LOGBACK_FILE"
 [[ $JAVA_OPTS != *-Dlogback.configuration* && -f $DEFAULT_LOGBACK_FILE ]] && JAVA_OPTS="$JAVA_OPTS -Dlogback.configuration=file:$DEFAULT_LOGBACK_FILE"
 
 
-JAVA_OPTS="$JAVA_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006"
-
 echo $JAVA $JAVA_OPTS -cp "$CLASSPATH" "$@"
 echo $JAVA $JAVA_OPTS -cp "$CLASSPATH" "$@"
 exec $JAVA $JAVA_OPTS -cp "$CLASSPATH" "$@"
 exec $JAVA $JAVA_OPTS -cp "$CLASSPATH" "$@"

+ 1 - 1
apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/AppMaster.scala

@@ -15,7 +15,7 @@ object AppMaster extends Logging {
   val SESSION_KIND = "livy-repl.session.kind"
   val SESSION_KIND = "livy-repl.session.kind"
 
 
   def main(args: Array[String]): Unit = {
   def main(args: Array[String]): Unit = {
-    val lang = args(1)
+    val lang = args(0)
 
 
     val containerIdString = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString)
     val containerIdString = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString)
     info("got container id: %s" format containerIdString)
     info("got container id: %s" format containerIdString)

+ 3 - 3
apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/Client.scala

@@ -13,8 +13,8 @@ import scala.collection.JavaConversions._
 object Client extends Logging {
 object Client extends Logging {
 
 
   def main(args: Array[String]): Unit = {
   def main(args: Array[String]): Unit = {
-    val packagePath = new Path(args(1))
-    val lang = args(2)
+    val packagePath = new Path(args(0))
+    val lang = args(1)
 
 
     val yarnConf = new YarnConfiguration()
     val yarnConf = new YarnConfiguration()
     yarnConf.set("yarn.resourcemanager.am.max-attempts", "1")
     yarnConf.set("yarn.resourcemanager.am.max-attempts", "1")
@@ -35,7 +35,7 @@ object Client extends Logging {
 
 
       info("waiting for job to start")
       info("waiting for job to start")
 
 
-      job.waitForStatus(Running(), 10000) match {
+      job.waitForStatus(Running(), 100000) match {
         case Some(Running()) => {
         case Some(Running()) => {
           info("job started successfully on %s:%s" format(job.getHost, job.getPort))
           info("job started successfully on %s:%s" format(job.getHost, job.getPort))
         }
         }