Browse Source

[spark] Initial version of sparker-yarn working

Erick Tryzelaar 11 years ago
parent
commit
a79eeee

+ 11 - 2
apps/spark/java/pom.xml

@@ -32,8 +32,8 @@
     <packaging>pom</packaging>
     <version>3.7.0-SNAPSHOT</version>
 
-    <name>Spark Main</name>
-    <description>Spark Main</description>
+    <name>sparker-main</name>
+    <description>sparker-main</description>
 
     <licenses>
         <license>
@@ -212,6 +212,15 @@
                     <target>1.6</target>
                 </configuration>
             </plugin>
+
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptor>src/main/assembly/dist.xml</descriptor>
+                </configuration>
+            </plugin>
+
             <!--
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>

+ 10 - 0
apps/spark/java/sparker-repl/pom.xml

@@ -94,6 +94,7 @@
                 </configuration>
             </plugin>
 
+            <!--
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-dependency-plugin</artifactId>
@@ -128,6 +129,15 @@
                     </archive>
                 </configuration>
             </plugin>
+            -->
+
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptor>src/main/assembly/dist.xml</descriptor>
+                </configuration>
+            </plugin>
 
             <!--
             <plugin>

+ 37 - 0
apps/spark/java/sparker-repl/src/main/assembly/dist.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  you under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+    <id>dist</id>
+    <formats>
+        <format>tar.gz</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>lib</outputDirectory>
+            <useProjectArtifact>false</useProjectArtifact>
+        </dependencySet>
+    </dependencySets>
+
+    <fileSets>
+        <fileSet>
+            <directory>${project.build.directory}</directory>
+            <outputDirectory>/lib</outputDirectory>
+            <includes>
+                <include>*.jar</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+</assembly>

+ 51 - 0
apps/spark/java/sparker-yarn/src/main/assembly/dist.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  you under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+    <id>dist</id>
+    <formats>
+        <format>tar.gz</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>lib</outputDirectory>
+            <useProjectArtifact>false</useProjectArtifact>
+            <!--
+            <excludes>
+                <exclude>commons-lang:commons-lang</exclude>
+                <exclude>log4j:log4j</exclude>
+            </excludes>
+            -->
+        </dependencySet>
+    </dependencySets>
+
+    <fileSets>
+        <fileSet>
+            <directory>${basedir}/src/main/bash</directory>
+            <outputDirectory>/bin</outputDirectory>
+            <fileMode>0744</fileMode>
+            <includes>
+                <include>*</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.build.directory}</directory>
+            <outputDirectory>/lib</outputDirectory>
+            <includes>
+                <include>*.jar</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+</assembly>

+ 21 - 0
apps/spark/java/sparker-yarn/src/main/bash/run-am.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[[ $JAVA_OPTS != *-server* ]] && export JAVA_OPTS="$JAVA_OPTS -server"
+
+exec $(dirname $0)/run-class.sh com.cloudera.hue.sparker.yarn.AppMaster $@

+ 36 - 0
apps/spark/java/sparker-yarn/src/main/bash/run-class.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+home_dir=`pwd`
+base_dir=$(dirname $0)/..
+cd $base_dir
+base_dir=`pwd`
+cd $home_dir
+
+HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HOME/.sparker}"
+HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
+CLASSPATH=$HADOOP_CONF_DIR
+
+for file in $base_dir/lib/*.[jw]ar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+if [ -z "$JAVA_HOME" ]; then
+  JAVA="java"
+else
+  JAVA="$JAVA_HOME/bin/java"
+fi
+
+# Try and use 64-bit mode if available in JVM_OPTS
+function check_and_enable_64_bit_mode {
+  `$JAVA -d64 -version`
+  if [ $? -eq 0 ] ; then
+    JAVA_OPTS="$JAVA_OPTS -d64"
+  fi
+}
+
+# Check if 64 bit is set. If not - try and set it if it's supported
+[[ $JAVA_OPTS != *-d64* ]] && check_and_enable_64_bit_mode
+
+echo $JAVA $JAVA_OPTS -cp $CLASSPATH $@
+exec $JAVA $JAVA_OPTS -cp $CLASSPATH $@

+ 19 - 0
apps/spark/java/sparker-yarn/src/main/bash/run-job.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+exec $(dirname $0)/run-class.sh com.cloudera.hue.sparker.yarn.Client $@

+ 30 - 0
apps/spark/java/sparker-yarn/src/main/resources/log4j.properties

@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.err
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n
+
+log4j.rootLogger=INFO, stdout
+
+# Switching off most of Apache DS logqing which is QUITE verbose
+log4j.logger.org.apache.directory=OFF
+log4j.logger.org.apache.directory.server.kerberos=INFO, stdout
+log4j.additivity.org.apache.directory=false
+log4j.logger.net.sf.ehcache=INFO, stdout

+ 49 - 0
apps/spark/java/sparker-yarn/src/main/scala/com/cloudera/hue/sparker/yarn/AppMaster.scala

@@ -0,0 +1,49 @@
+package com.cloudera.hue.sparker.yarn
+
+import org.apache.hadoop.net.NetUtils
+import org.apache.hadoop.yarn.api.ApplicationConstants
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.util.ConverterUtils
+
+object AppMaster extends Logging {
+
+  def main(args: Array[String]): Unit = {
+    val containerIdString = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString)
+    info("got container id: %s" format containerIdString)
+    val containerId = ConverterUtils.toContainerId(containerIdString)
+
+    val appAttemptId = containerId.getApplicationAttemptId
+    info("got app attempt id: %s" format containerIdString)
+
+    val nodeHostString = System.getenv(ApplicationConstants.Environment.NM_HOST.toString)
+    info("got node manager host: %s" format nodeHostString)
+
+    val nodePortString = System.getenv(ApplicationConstants.Environment.NM_PORT.toString)
+    info("got node manager port: %s" format nodeHostString)
+
+    val yarnConfig = new YarnConfiguration
+    val amRMClient = AMRMClient.createAMRMClient()
+    amRMClient.init(yarnConfig)
+    amRMClient.start()
+
+    val appMasterHostname = NetUtils.getHostname
+    val appMasterRpcPort = -1
+    val appMasterTrackingUrl = ""
+
+    val response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl)
+
+    val maxMem = response.getMaximumResourceCapability.getMemory
+    info("max mem capacity on this cluster: %s" format maxMem)
+
+    val maxVCores = response.getMaximumResourceCapability.getVirtualCores
+    info("max vcore capacity on this cluster: %s" format maxMem)
+
+    val appStatus = FinalApplicationStatus.SUCCEEDED
+    val appMessage = "wee"
+    amRMClient.unregisterApplicationMaster(appStatus, appMessage, null)
+    amRMClient.stop()
+  }
+
+}

+ 172 - 14
apps/spark/java/sparker-repl/src/main/scala/com/cloudera/hue/sparker/repl/yarn/Client.scala → apps/spark/java/sparker-yarn/src/main/scala/com/cloudera/hue/sparker/yarn/Client.scala

@@ -1,50 +1,123 @@
 package com.cloudera.hue.sparker.yarn
 
-import java.util.Collections
-
-import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.hadoop.yarn.api.ApplicationConstants
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
 import org.apache.hadoop.yarn.api.records._
 import org.apache.hadoop.yarn.client.api.YarnClient
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.hadoop.yarn.util.{ConverterUtils, Records}
+import org.slf4j.LoggerFactory
 
 import scala.collection.JavaConversions._
 
-class Client {
-  def main(args: Array[String]) = {
+object Client extends Logging {
+
+  def main(args: Array[String]): Unit = {
+    println(args.length)
+    args.foreach(println(_))
 
     val packagePath = new Path(args(1))
 
     val yarnConf = new YarnConfiguration()
-    val yarnClient = YarnClient.createYarnClient()
-    yarnClient.init(yarnConf)
-    yarnClient.start
+    val client = new Client(yarnConf)
 
     try {
-      submitApplication(yarnClient, yarnConf, packagePath, List("echo hi"))
+      val job = client.submitApplication(
+        packagePath,
+        List(
+          "__package/bin/run-am.sh 1>%s/stdout 2>%s/stderr" format (
+            ApplicationConstants.LOG_DIR_EXPANSION_VAR,
+            ApplicationConstants.LOG_DIR_EXPANSION_VAR
+          )
+        /*
+          "/bin/pwd " +
+            " 1>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
+            " 2>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr;",
+          "/bin/ls " +
+            " 1>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
+            " 2>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr;",
+          "/bin/echo hi" +
+          " 1>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
+          " 2>>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
+          */
+        )
+      )
+
+      info("waiting for job to start")
+
+      job.waitForStatus(Running(), 500) match {
+        case Some(Running()) => {
+          info("job started successfully")
+        }
+        case Some(appStatus) => {
+          warn("unable to start job successfully. job has status %s" format appStatus)
+        }
+        case None => {
+          warn("timed out waiting for job to start")
+        }
+      }
+
+      job.waitForFinish(100000) match {
+        case Some(SuccessfulFinish()) => {
+          info("job finished successfully")
+        }
+        case Some(appStatus) => {
+          info("job finished unsuccessfully %s" format appStatus)
+        }
+        case None => {
+          info("timed out")
+        }
+      }
+
     } finally {
-      yarnClient.close()
+      client.close()
     }
   }
+}
+
+class Client(yarnConf: YarnConfiguration) {
+
+  import Client._
 
-  def submitApplication(yarnClient: YarnClient, yarnConf: YarnConfiguration, packagePath: Path, cmds: List[String]) = {
+  val yarnClient = YarnClient.createYarnClient()
+  yarnClient.init(yarnConf)
+  yarnClient.start()
+
+  def submitApplication(packagePath: Path, cmds: List[String]): Job = {
     val app = yarnClient.createApplication()
     val newAppResponse = app.getNewApplicationResponse
 
     val appId = newAppResponse.getApplicationId
 
+    info("preparing to submit %s" format appId)
+
     val appContext = app.getApplicationSubmissionContext
+    appContext.setApplicationName(appId.toString)
+
     val containerCtx = Records.newRecord(classOf[ContainerLaunchContext])
     val resource = Records.newRecord(classOf[Resource])
-    val packageResource = Records.newRecord(classOf[LocalResource])
 
-    appContext.setApplicationName(appId.toString)
+    info("Copy app master jar from local filesystem and add to the local environment")
+    /*
+    val localResources = Map(
+      "app" => uploadLocalResource()
+    )
+    Map
+
+
+    addToLocalResources(fs, appMasterJar, appMasterJarPath, appId, localResources, null)
+    */
+
+    val packageResource = Records.newRecord(classOf[LocalResource])
 
     val packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath)
     val fileStatus = packagePath.getFileSystem(yarnConf).getFileStatus(packagePath)
 
     packageResource.setResource(packageUrl)
+    info("set package url to %s for %s" format (packageUrl, appId))
     packageResource.setSize(fileStatus.getLen)
+    info("set package size to %s for %s" format (fileStatus.getLen, appId))
     packageResource.setTimestamp(fileStatus.getModificationTime)
     packageResource.setType(LocalResourceType.ARCHIVE)
     packageResource.setVisibility(LocalResourceVisibility.APPLICATION)
@@ -54,14 +127,99 @@ class Client {
     appContext.setResource(resource)
 
     containerCtx.setCommands(cmds)
-    containerCtx.setLocalResources(Collections.singletonMap("__package", packageResource))
+    containerCtx.setLocalResources(Map("__package" -> packageResource))
+
     appContext.setApplicationId(appId)
     appContext.setAMContainerSpec(containerCtx)
     appContext.setApplicationType("sparker")
+
+    info("submitting application request for %s" format appId)
+
     yarnClient.submitApplication(appContext)
+
+    new Job(yarnClient, appId)
+  }
+
+  def close(): Unit = {
+    yarnClient.close()
+  }
+
+  private def addToLocalResources(fs: FileSystem, fileSrcPath: String, fileDstPath: String, appId: String): LocalResource = {
+    val appName = "sparker"
+    val suffix = appName + "/" + appId + "/" + fileDstPath
+
+    val dst = new Path(fs.getHomeDirectory, suffix)
+
+    fs.copyFromLocalFile(new Path(fileSrcPath), dst)
+
+    val srcFileStatus = fs.getFileStatus(dst)
+    LocalResource.newInstance(
+      ConverterUtils.getYarnUrlFromURI(dst.toUri),
+      LocalResourceType.FILE,
+      LocalResourceVisibility.APPLICATION,
+      srcFileStatus.getLen,
+      srcFileStatus.getModificationTime
+    )
   }
 }
 
+class Job(client: YarnClient, appId: ApplicationId) {
+
+  def waitForFinish(timeoutMs: Long): Option[ApplicationStatus] = {
+    val startTimeMs = System.currentTimeMillis()
+
+    while (System.currentTimeMillis() - startTimeMs < timeoutMs) {
+      val status = getStatus()
+      status match {
+        case SuccessfulFinish() | UnsuccessfulFinish() => {
+          return Some(status)
+        }
+        case _ =>
+      }
+
+      Thread.sleep(1000)
+    }
+
+    None
+  }
+
+  def waitForStatus(status: ApplicationStatus, timeoutMs: Long): Option[ApplicationStatus] = {
+    val startTimeMs = System.currentTimeMillis()
+
+    while (System.currentTimeMillis() - startTimeMs < timeoutMs) {
+      if (getStatus() == status) {
+        return Some(status)
+      }
+
+      Thread.sleep(1000)
+    }
+
+    None
+  }
+
+  private def getStatus(): ApplicationStatus = {
+    val statusResponse = client.getApplicationReport(appId)
+    convertState(statusResponse.getYarnApplicationState, statusResponse.getFinalApplicationStatus)
+  }
+
+  private def convertState(state: YarnApplicationState, status: FinalApplicationStatus): ApplicationStatus = {
+    (state, status) match {
+      case (YarnApplicationState.FINISHED, FinalApplicationStatus.SUCCEEDED) => SuccessfulFinish()
+      case (YarnApplicationState.KILLED, _) | (YarnApplicationState.FAILED, _) => UnsuccessfulFinish()
+      case (YarnApplicationState.NEW, _) | (YarnApplicationState.SUBMITTED, _) => New()
+      case _ => Running()
+    }
+  }
+}
+
+trait ApplicationStatus
+case class New() extends ApplicationStatus
+case class Running() extends ApplicationStatus
+case class SuccessfulFinish() extends ApplicationStatus
+case class UnsuccessfulFinish() extends ApplicationStatus
+
+
+
 
 /*
 object Client {

+ 28 - 0
apps/spark/java/sparker-yarn/src/main/scala/com/cloudera/hue/sparker/yarn/Logging.scala

@@ -0,0 +1,28 @@
+package com.cloudera.hue.sparker.yarn
+
+import org.slf4j.LoggerFactory
+
+trait Logging {
+  val loggerName = this.getClass.getName
+  lazy val logger = LoggerFactory.getLogger(loggerName)
+
+  def debug(message: => Any) = {
+    if (logger.isDebugEnabled) {
+      logger.debug(message.toString)
+    }
+  }
+
+  def info(message: => Any) = {
+    if (logger.isInfoEnabled) {
+      logger.info(message.toString)
+    }
+  }
+
+  def warn(message: => Any) = {
+    logger.warn(message.toString)
+  }
+
+  def error(message: => Any) = {
+    logger.error(message.toString)
+  }
+}

+ 1 - 1
apps/spark/java/sparker-repl/src/main/scala/com/cloudera/hue/sparker/repl/yarn/YarnJob.scala → apps/spark/java/sparker-yarn/src/main/scala/com/cloudera/hue/sparker/yarn/YarnJob.scala

@@ -4,6 +4,6 @@ import org.apache.hadoop.conf.Configuration
 
 class YarnJob(config: Configuration) {
 
-  val client = new Client()
+  //val client = new Client
 
 }

+ 65 - 0
apps/spark/java/src/main/assembly/dist.xml

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  you under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+    <id>dist</id>
+    <formats>
+        <format>tar.gz</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>lib</outputDirectory>
+            <useProjectArtifact>false</useProjectArtifact>
+            <!--
+            <excludes>
+                <exclude>commons-lang:commons-lang</exclude>
+                <exclude>log4j:log4j</exclude>
+            </excludes>
+            -->
+        </dependencySet>
+    </dependencySets>
+
+    <moduleSets>
+        <moduleSet>
+            <includes>
+                <!--
+                <include>com.cloudera.hue.sparker:sparker-repl</include>
+                <include>com.cloudera.hue.sparker:sparker-server</include>
+                -->
+                <include>com.cloudera.hue.sparker:sparker-yarn</include>
+            </includes>
+        </moduleSet>
+    </moduleSets>
+
+    <!--
+    <fileSets>
+        <fileSet>
+            <directory>${basedir}/src/main/bash</directory>
+            <outputDirectory>/bin</outputDirectory>
+            <fileMode>0744</fileMode>
+            <includes>
+                <include>*</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.build.directory}</directory>
+            <outputDirectory>/lib</outputDirectory>
+            <includes>
+                <include>*.jar</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+    -->
+</assembly>