Sfoglia il codice sorgente

[spark] Fix some more renames

Erick Tryzelaar 11 anni fa
parent
commit
ad958fd

+ 1 - 1
apps/spark/java/livy-server/src/main/java/com/cloudera/hue/livy/server/sessions/SparkSession.java

@@ -52,7 +52,7 @@ public class SparkSession implements Session {
     private static final Logger LOG = LoggerFactory.getLogger(SparkSession.class);
 
     private static final String SPARKER_HOME = System.getenv("SPARKER_HOME");
-    private static final String SPARKER_SHELL = SPARKER_HOME + "/sparker-shell";
+    private static final String SPARKER_SHELL = SPARKER_HOME + "/livy-shell";
 
     private final String id;
     private State state = State.READY;

+ 32 - 0
apps/spark/java/livy-yarn/pom.xml

@@ -28,6 +28,26 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${hadoop.version}</version>
+            <!--
+            <exclusions>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.mortbay.jetty</groupId>
+                    <artifactId>jetty</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.mortbay.jetty</groupId>
+                    <artifactId>jetty-util</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+            </exclusions>
+            -->
         </dependency>
 
         <dependency>
@@ -52,6 +72,18 @@
             <groupId>com.cloudera.hue.livy</groupId>
             <artifactId>livy-repl</artifactId>
             <version>${project.version}</version>
+            <!--
+            <exclusions>
+                <exclusion>
+                    <groupId>org.scala-lang</groupId>
+                    <artifactId>scala-compiler</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+            </exclusions>
+            -->
         </dependency>
 
     </dependencies>

+ 16 - 1
apps/spark/java/livy-yarn/src/main/assembly/dist.xml

@@ -22,7 +22,13 @@
     <dependencySets>
         <dependencySet>
             <outputDirectory>lib</outputDirectory>
-            <useProjectArtifact>false</useProjectArtifact>
+            <useProjectArtifact>true</useProjectArtifact>
+
+            <!--
+            <excludes>
+                <exclude></exclude>
+            </excludes>
+            -->
         </dependencySet>
     </dependencySets>
 
@@ -35,6 +41,15 @@
                 <include>*</include>
             </includes>
         </fileSet>
+
+        <fileSet>
+            <directory>${basedir}/src/main/resources</directory>
+            <outputDirectory>/resources</outputDirectory>
+            <fileMode>0744</fileMode>
+            <includes>
+                <include>*</include>
+            </includes>
+        </fileSet>
     </fileSets>
 
 </assembly>

+ 11 - 7
apps/spark/java/livy-yarn/src/main/bash/run-class.sh

@@ -8,12 +8,13 @@ cd $home_dir
 
 HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HOME/.livy}"
 HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
-CLASSPATH=$HADOOP_CONF_DIR
+CLASSPATH="$HADOOP_CONF_DIR:$base_dir/lib/*"
+DEFAULT_LOG4J_FILE="$base_dir/resources/log4j.properties"
 
-for file in $base_dir/lib/*.[jw]ar;
-do
-  CLASSPATH=$CLASSPATH:$file
-done
+#for file in $base_dir/lib/*.[jw]ar;
+#do
+#  CLASSPATH=$CLASSPATH:$file
+#done
 
 if [ -z "$JAVA_HOME" ]; then
   JAVA="java"
@@ -32,5 +33,8 @@ function check_and_enable_64_bit_mode {
 # Check if 64 bit is set. If not - try and set it if it's supported
 [[ $JAVA_OPTS != *-d64* ]] && check_and_enable_64_bit_mode
 
-echo $JAVA $JAVA_OPTS -cp $CLASSPATH $@
-exec $JAVA $JAVA_OPTS -cp $CLASSPATH $@
+# Check if log4j configuration is specified. If not - set to lib/log4j.xml
+[[ $JAVA_OPTS != *-Dlog4j.configuration* && -f $DEFAULT_LOG4J_FILE ]] && JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$DEFAULT_LOG4J_FILE"
+
+echo $JAVA $JAVA_OPTS -cp "$CLASSPATH" "$@"
+exec $JAVA $JAVA_OPTS -cp "$CLASSPATH" "$@"

+ 6 - 5
apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/AppMaster.scala

@@ -6,9 +6,9 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
 import org.apache.hadoop.yarn.client.api.AMRMClient
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.hadoop.yarn.util.ConverterUtils
-import org.mortbay.jetty.Server
-import org.mortbay.jetty.servlet.DefaultServlet
-import org.mortbay.jetty.webapp.WebAppContext
+import org.eclipse.jetty.server.Server
+import org.eclipse.jetty.servlet.DefaultServlet
+import org.eclipse.jetty.webapp.WebAppContext
 import org.scalatra.servlet.{AsyncSupport, ScalatraListener}
 
 import scala.concurrent.ExecutionContext
@@ -52,10 +52,11 @@ object AppMaster extends Logging {
 
       // Now that the server is up and running register it with YARN.
       val appMasterHostname = NetUtils.getHostname
-      val appMasterRpcPort = -1
+      val appMasterRpcPort = server.getConnectors()(0).getLocalPort
       val appMasterTrackingUrl = ""
 
-      val port =  server.getConnectors()(0).getLocalPort
+      info("Starting RPC server on %s:%s" format(appMasterHostname, appMasterRpcPort))
+      info("Tracking URL: %s" format appMasterTrackingUrl)
 
       val response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl)
 

+ 4 - 2
apps/spark/java/livy-yarn/src/main/scala/com/cloudera/hue/livy/yarn/Client.scala

@@ -20,6 +20,8 @@ object Client extends Logging {
     val packagePath = new Path(args(1))
 
     val yarnConf = new YarnConfiguration()
+    yarnConf.set("yarn.resourcemanager.am.max-attempts", "1")
+
     val client = new Client(yarnConf)
 
     try {
@@ -228,7 +230,7 @@ object Client {
       amContainer.setCommands(
         Collections.singletonList(
           "$JAVA_HOME/bin/java" +
-            " com.cloudera.hue.sparker.repl.yarn.ApplicationMaster" +
+            " com.cloudera.hue.livy.repl.yarn.ApplicationMaster" +
             " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
             " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"))
 
@@ -302,7 +304,7 @@ object Client {
       val appContext = yarnClient.createApplication.getApplicationSubmissionContext
       val appId = appContext.getApplicationId
 
-      val appName = "sparker-repl"
+      val appName = "livy-repl"
       val amPriority = 0
       val amQueue = "default"
 

+ 0 - 65
apps/spark/java/src/main/assembly/dist3.xml

@@ -1,65 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>dist</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <useProjectArtifact>false</useProjectArtifact>
-            <!--
-            <excludes>
-                <exclude>commons-lang:commons-lang</exclude>
-                <exclude>log4j:log4j</exclude>
-            </excludes>
-            -->
-        </dependencySet>
-    </dependencySets>
-
-    <moduleSets>
-        <moduleSet>
-            <includes>
-                <!--
-                <include>com.cloudera.hue.sparker:sparker-repl</include>
-                <include>com.cloudera.hue.sparker:sparker-server</include>
-                -->
-                <include>com.cloudera.hue.sparker:sparker-yarn</include>
-            </includes>
-        </moduleSet>
-    </moduleSets>
-
-    <!--
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}/src/main/bash</directory>
-            <outputDirectory>/bin</outputDirectory>
-            <fileMode>0744</fileMode>
-            <includes>
-                <include>*</include>
-            </includes>
-        </fileSet>
-        <fileSet>
-            <directory>${project.build.directory}</directory>
-            <outputDirectory>/lib</outputDirectory>
-            <includes>
-                <include>*.jar</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    -->
-</assembly>