Эх сурвалжийг харах

[livy] Fix spark creating a spark context

Problem was we were including an incompatible version of akka.
Erick Tryzelaar 10 жил өмнө
parent
commit
c603f1e

+ 23 - 5
apps/spark/java/livy-core/pom.xml

@@ -31,7 +31,6 @@
             <groupId>org.eclipse.jetty</groupId>
             <artifactId>jetty-server</artifactId>
             <version>${jetty.version}</version>
-            <scope>container</scope>
         </dependency>
 
         <dependency>
@@ -44,6 +43,12 @@
             <groupId>org.scalatra</groupId>
             <artifactId>scalatra_2.10</artifactId>
             <version>${scalatra.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.typesafe.akka</groupId>
+                    <artifactId>akka-actor_2.10</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
@@ -52,16 +57,29 @@
             <version>${scalatra.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.scalatra</groupId>
+            <artifactId>scalatra-scalatest_2.10</artifactId>
+            <version>${scalatra.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.scalatra</groupId>
+            <artifactId>scalatra-json_2.10</artifactId>
+            <version>${scalatra.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.json4s</groupId>
-            <artifactId>json4s-jackson_2.10</artifactId>
+            <artifactId>json4s_2.10</artifactId>
             <version>${json4s.version}</version>
         </dependency>
 
         <dependency>
-            <groupId>org.eclipse.jetty</groupId>
-            <artifactId>jetty-server</artifactId>
-            <version>${jetty.version}</version>
+            <groupId>org.json4s</groupId>
+            <artifactId>json4s-jackson_2.10</artifactId>
+            <version>${json4s.version}</version>
         </dependency>
 
     </dependencies>

+ 11 - 36
apps/spark/java/livy-repl/pom.xml

@@ -20,6 +20,12 @@
 
     <dependencies>
 
+        <dependency>
+            <groupId>com.cloudera.hue.livy</groupId>
+            <artifactId>livy-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-repl_2.10</artifactId>
@@ -29,6 +35,11 @@
                     <groupId>org.eclipse.jetty</groupId>
                     <artifactId>jetty-server</artifactId>
                 </exclusion>
+
+                <exclusion>
+                    <groupId>org.eclipse.jetty</groupId>
+                    <artifactId>jetty-util</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -38,18 +49,6 @@
             <version>${json4s.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>org.scalatra</groupId>
-            <artifactId>scalatra_2.10</artifactId>
-            <version>${scalatra.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.scalatra</groupId>
-            <artifactId>scalatra-json_2.10</artifactId>
-            <version>${scalatra.version}</version>
-        </dependency>
-
         <dependency>
             <groupId>org.scalatra</groupId>
             <artifactId>scalatra-scalatest_2.10</artifactId>
@@ -71,18 +70,6 @@
             <scope>provided</scope>
         </dependency>
 
-        <dependency>
-            <groupId>com.cloudera.hue.livy</groupId>
-            <artifactId>livy-core</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>ch.qos.logback</groupId>
-            <artifactId>logback-access</artifactId>
-            <version>${logback.version}</version>
-        </dependency>
-
     </dependencies>
 
     <build>
@@ -170,16 +157,4 @@
         </plugins>
     </build>
 
-    <reporting>
-        <plugins>
-            <plugin>
-                <groupId>org.scala-tools</groupId>
-                <artifactId>maven-scala-plugin</artifactId>
-                <configuration>
-                    <scalaVersion>${scala.version}</scalaVersion>
-                </configuration>
-            </plugin>
-        </plugins>
-    </reporting>
-
 </project>

+ 2 - 2
apps/spark/java/livy-repl/src/test/scala/com/cloudera/hue/livy/repl/SparkSessionSpec.scala

@@ -1,9 +1,9 @@
 package com.cloudera.hue.livy.repl
 
 import com.cloudera.hue.livy.repl.scala.SparkSession
-import org.json4s.{Extraction, DefaultFormats}
-import org.scalatest.{BeforeAndAfter, FunSpec}
+import org.json4s.{DefaultFormats, Extraction}
 import org.scalatest.matchers.ShouldMatchers
+import org.scalatest.{FunSpec, BeforeAndAfter}
 
 import _root_.scala.concurrent.Await
 import _root_.scala.concurrent.duration.Duration

+ 1 - 2
apps/spark/java/livy-repl/src/test/scala/com/cloudera/hue/livy/repl/WebAppSpec.scala

@@ -1,13 +1,12 @@
 package com.cloudera.hue.livy.repl
 
 import com.cloudera.hue.livy.repl.Session.State
-import org.json4s.{Extraction, DefaultFormats, JValue}
 import org.json4s.JsonAST.{JArray, JString}
 import org.json4s.JsonDSL._
 import org.json4s.jackson.JsonMethods._
+import org.json4s.{DefaultFormats, Extraction, JValue}
 import org.scalatest.{BeforeAndAfter, FunSpec}
 import org.scalatra.test.scalatest.ScalatraSuite
-
 import _root_.scala.concurrent.Future
 
 class WebAppSpec extends ScalatraSuite with FunSpec with BeforeAndAfter {

+ 6 - 26
apps/spark/java/livy-server/pom.xml

@@ -32,32 +32,6 @@
 
     <dependencies>
 
-        <dependency>
-            <groupId>org.json4s</groupId>
-            <artifactId>json4s_2.10</artifactId>
-            <version>${json4s.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.json4s</groupId>
-            <artifactId>json4s-jackson_2.10</artifactId>
-            <version>${json4s.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.scalatra</groupId>
-            <artifactId>scalatra_2.10</artifactId>
-            <version>${scalatra.version}</version>
-            <scope>compile</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.scalatra</groupId>
-            <artifactId>scalatra-json_2.10</artifactId>
-            <version>${scalatra.version}</version>
-            <scope>compile</scope>
-        </dependency>
-
         <dependency>
             <groupId>com.cloudera.hue.livy</groupId>
             <artifactId>livy-core</artifactId>
@@ -76,6 +50,12 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>org.json4s</groupId>
+            <artifactId>json4s-jackson_2.10</artifactId>
+            <version>${json4s.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>net.databinder.dispatch</groupId>
             <artifactId>dispatch-core_2.10</artifactId>

+ 15 - 32
apps/spark/java/pom.xml

@@ -51,10 +51,9 @@
         <javaVersion>1.7</javaVersion>
         <hadoop.version>2.5.0</hadoop.version>
         <guava.version>11.0.2</guava.version>
-        <jetty.version>8.1.16.v20140903</jetty.version>
+        <jetty.version>8.1.14.v20131031</jetty.version>
         <json4s.version>3.2.11</json4s.version>
         <logback.version>1.1.2</logback.version>
-        <scala.binary.version>2.10.3</scala.binary.version>
         <scala.version>2.10.3</scala.version>
         <scalatra.version>2.2.1</scalatra.version>
         <spark.version>1.1.0</spark.version>
@@ -171,34 +170,6 @@
                     <version>2.4.0</version>
                 </plugin>
 
-                <!-- disable surefire -->
-                <plugin>
-                    <groupId>org.apache.maven.plugins</groupId>
-                    <artifactId>maven-surefire-plugin</artifactId>
-                    <version>2.7</version>
-                    <configuration>
-                        <skipTests>true</skipTests>
-                    </configuration>
-                </plugin>
-                <!-- enable scalatest -->
-                <plugin>
-                    <groupId>org.scalatest</groupId>
-                    <artifactId>scalatest-maven-plugin</artifactId>
-                    <version>1.0</version>
-                    <configuration>
-                        <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
-                        <junitxml>.</junitxml>
-                        <filereports>WDF TestSuite.txt</filereports>
-                    </configuration>
-                    <executions>
-                        <execution>
-                            <id>test</id>
-                            <goals>
-                                <goal>test</goal>
-                            </goals>
-                        </execution>
-                    </executions>
-                </plugin>
             </plugins>
         </pluginManagement>
 
@@ -248,8 +219,8 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <configuration>
-                    <source>1.6</source>
-                    <target>1.6</target>
+                    <source>${javaVersion}</source>
+                    <target>${javaVersion}</target>
                 </configuration>
             </plugin>
 
@@ -301,4 +272,16 @@
 
     </build>
 
+    <reporting>
+        <plugins>
+            <plugin>
+                <groupId>org.scala-tools</groupId>
+                <artifactId>maven-scala-plugin</artifactId>
+                <configuration>
+                    <scalaVersion>${scala.version}</scalaVersion>
+                </configuration>
+            </plugin>
+        </plugins>
+    </reporting>
+
 </project>