فهرست منبع

[livy] Skip sparkR tests if sparkR executable is not present

This touches a lot of the tests in order to use the scalatest
FlatSpec `withFixture` in order to skip the tests.
Erick Tryzelaar 10 سال پیش
والد
کامیت
ca58ba7d57

+ 21 - 4
apps/spark/java/livy-repl/src/main/scala/com/cloudera/hue/livy/repl/sparkr/SparkRInterpreter.scala

@@ -60,11 +60,10 @@ object SparkRInterpreter {
     ).r.unanchored
 
   def apply(): SparkRInterpreter = {
-    val sparkrExec = sys.env.getOrElse("SPARKR_DRIVER_R", "sparkR")
+    val executable = sparkRExecutable
+      .getOrElse(throw new Exception(f"Cannot find sparkR executable"))
 
-    val builder = new ProcessBuilder(Seq(
-      sparkrExec
-    ))
+    val builder = new ProcessBuilder(Seq(executable.getAbsolutePath))
 
     val env = builder.environment()
     env.put("SPARK_HOME", sys.env.getOrElse("SPARK_HOME", "."))
@@ -77,6 +76,24 @@ object SparkRInterpreter {
     new SparkRInterpreter(process)
   }
 
+  def sparkRExecutable: Option[File] = {
+    val executable = sys.env.getOrElse("SPARKR_DRIVER_R", "sparkR")
+    val executableFile = new File(executable)
+
+    if (executableFile.exists) {
+      Some(executableFile)
+    } else {
+      // see if sparkR is on the path.
+      val path: Option[String] = sys.env.get("PATH")
+      assume(path.isDefined, "PATH is not defined?")
+
+      path.get
+        .split(File.pathSeparator)
+        .map(new File(_, executable))
+        .find(_.exists)
+    }
+  }
+
   private def createFakeShell(): File = {
     val source = getClass.getClassLoader.getResourceAsStream("fake_R.sh")
 

+ 7 - 10
apps/spark/java/livy-repl/src/test/scala/com/cloudera/hue/livy/repl/BaseInterpreterSpec.scala

@@ -18,20 +18,17 @@
 
 package com.cloudera.hue.livy.repl
 
-import org.scalatest.{BeforeAndAfter, Matchers, FunSpec}
+import org.scalatest.{FlatSpec, Matchers}
 
-abstract class BaseInterpreterSpec extends FunSpec with Matchers with BeforeAndAfter {
+abstract class BaseInterpreterSpec extends FlatSpec with Matchers {
 
   def createInterpreter(): Interpreter
 
-  var interpreter: Interpreter = null
-
-  before {
-    interpreter = createInterpreter()
+  def withInterpreter(testCode: Interpreter => Any) = {
+    val interpreter = createInterpreter()
     interpreter.start()
-  }
-
-  after {
-    interpreter.close()
+    try {
+      testCode(interpreter)
+    } finally interpreter.close()
   }
 }

+ 16 - 20
apps/spark/java/livy-repl/src/test/scala/com/cloudera/hue/livy/repl/BaseSessionSpec.scala

@@ -20,36 +20,32 @@ package com.cloudera.hue.livy.repl
 
 import java.util.concurrent.TimeUnit
 
-import com.cloudera.hue.livy.sessions.{Idle, Starting}
+import com.cloudera.hue.livy.sessions.{NotStarted, Idle, Starting}
 import org.json4s.DefaultFormats
-import org.scalatest.{Matchers, FunSpec, BeforeAndAfter}
+import org.scalatest.{FlatSpec, Matchers}
 
 import _root_.scala.concurrent.duration.Duration
 
-abstract class BaseSessionSpec extends FunSpec with Matchers with BeforeAndAfter {
+abstract class BaseSessionSpec extends FlatSpec with Matchers {
 
   implicit val formats = DefaultFormats
 
-  def createInterpreter(): Interpreter
-
-  var session: Session = null
-
-  before {
-    session = Session(createInterpreter())
+  def withSession(testCode: Session => Any) = {
+    val session = Session(createInterpreter())
+    session.waitForStateChange(NotStarted(), Duration(30, TimeUnit.SECONDS))
+    try {
+      testCode(session)
+    } finally session.close()
   }
 
-  after {
-    session.close()
-  }
+  def createInterpreter(): Interpreter
 
-  describe("A session") {
-    it("should start in the starting or idle state") {
-      session.state should (equal (Starting()) or equal (Idle()))
-    }
+  it should "start in the starting or idle state" in withSession { session =>
+    session.state should (equal (Starting()) or equal (Idle()))
+  }
 
-    it("should eventually become the idle state") {
-      session.waitForStateChange(Starting(), Duration(30, TimeUnit.SECONDS))
-      session.state should equal (Idle())
-    }
+  it should "eventually become the idle state" in withSession { session =>
+    session.waitForStateChange(Starting(), Duration(30, TimeUnit.SECONDS))
+    session.state should equal (Idle())
   }
 }

+ 89 - 91
apps/spark/java/livy-repl/src/test/scala/com/cloudera/hue/livy/repl/PythonInterpreterSpec.scala

@@ -29,103 +29,101 @@ class PythonInterpreterSpec extends BaseInterpreterSpec {
 
   override def createInterpreter() = PythonInterpreter()
 
-  describe("A python interpreter") {
-    it("should execute `1 + 2` == 3") {
-      val response = interpreter.execute("1 + 2")
-      response should equal (Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "3"
-      ))
-    }
-
-    it("should execute multiple statements") {
-      var response = interpreter.execute("x = 1")
-      response should equal (Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> ""
-      ))
-
-      response = interpreter.execute("y = 2")
-      response should equal (Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> ""
-      ))
-
-      response = interpreter.execute("x + y")
-      response should equal (Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "3"
-      ))
-    }
-
-    it("should execute multiple statements in one block") {
-      val response = interpreter.execute(
-        """
-          |x = 1
-          |
-          |y = 2
-          |
-          |x + y
-        """.stripMargin)
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "3"
-      ))
-    }
-
-    it("should do table magic") {
-      val response = interpreter.execute(
-        """x = [[1, 'a'], [3, 'b']]
-          |%table x
-        """.stripMargin)
-
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.APPLICATION_LIVY_TABLE_JSON -> (
-          ("headers" -> List(
-            ("type" -> "INT_TYPE") ~ ("name" -> "0"),
-            ("type" -> "STRING_TYPE") ~ ("name" -> "1")
-          )) ~
+  it should "execute `1 + 2` == 3" in withInterpreter { interpreter =>
+    val response = interpreter.execute("1 + 2")
+    response should equal (Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "3"
+    ))
+  }
+
+  it should "execute multiple statements" in withInterpreter { interpreter =>
+    var response = interpreter.execute("x = 1")
+    response should equal (Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> ""
+    ))
+
+    response = interpreter.execute("y = 2")
+    response should equal (Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> ""
+    ))
+
+    response = interpreter.execute("x + y")
+    response should equal (Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "3"
+    ))
+  }
+
+  it should "execute multiple statements in one block" in withInterpreter { interpreter =>
+    val response = interpreter.execute(
+      """
+        |x = 1
+        |
+        |y = 2
+        |
+        |x + y
+      """.stripMargin)
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "3"
+    ))
+  }
+
+  it should "do table magic" in withInterpreter { interpreter =>
+    val response = interpreter.execute(
+      """x = [[1, 'a'], [3, 'b']]
+        |%table x
+      """.stripMargin)
+
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.APPLICATION_LIVY_TABLE_JSON -> (
+        ("headers" -> List(
+          ("type" -> "INT_TYPE") ~ ("name" -> "0"),
+          ("type" -> "STRING_TYPE") ~ ("name" -> "1")
+        )) ~
           ("data" -> List(
             List[JValue](1, "a"),
             List[JValue](3, "b")
           ))
         )
-      ))
-    }
-
-    it("should allow magic inside statements") {
-      val response = interpreter.execute(
-        """x = [[1, 'a'], [3, 'b']]
-          |%table x
-          |1 + 2
-        """.stripMargin)
-
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "3"
-      ))
-    }
-
-    it("should capture stdout") {
-      val response = interpreter.execute("print 'Hello World'")
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "Hello World"
-      ))
-    }
-
-    it("should report an error if accessing an unknown variable") {
-      val response = interpreter.execute("x")
-      response should equal(Interpreter.ExecuteError(
-        "NameError",
-        "name 'x' is not defined",
-        List(
-          "Traceback (most recent call last):\n",
-          "NameError: name 'x' is not defined\n"
-        )
-      ))
-    }
+    ))
+  }
+
+  it should "allow magic inside statements" in withInterpreter { interpreter =>
+    val response = interpreter.execute(
+      """x = [[1, 'a'], [3, 'b']]
+        |%table x
+        |1 + 2
+      """.stripMargin)
+
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "3"
+    ))
+  }
+
+  it should "capture stdout" in withInterpreter { interpreter =>
+    val response = interpreter.execute("print 'Hello World'")
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "Hello World"
+    ))
+  }
+
+  it should "report an error if accessing an unknown variable" in withInterpreter { interpreter =>
+    val response = interpreter.execute("x")
+    response should equal(Interpreter.ExecuteError(
+      "NameError",
+      "name 'x' is not defined",
+      List(
+        "Traceback (most recent call last):\n",
+        "NameError: name 'x' is not defined\n"
+      )
+    ))
+  }
 
-    it("should execute spark commands") {
-      val response = interpreter.execute(
-        """sc.parallelize(xrange(0, 2)).map(lambda i: i + 1).collect()""")
+  it should "execute spark commands" in withInterpreter { interpreter =>
+    val response = interpreter.execute(
+      """sc.parallelize(xrange(0, 2)).map(lambda i: i + 1).collect()""")
 
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "[1, 2]"
-      ))
-    }
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "[1, 2]"
+    ))
   }
 }

+ 157 - 159
apps/spark/java/livy-repl/src/test/scala/com/cloudera/hue/livy/repl/PythonSessionSpec.scala

@@ -29,178 +29,176 @@ class PythonSessionSpec extends BaseSessionSpec {
 
   override def createInterpreter() = PythonInterpreter()
 
-  describe("A python session") {
-    it("should execute `1 + 2` == 3") {
-      val statement = session.execute("1 + 2")
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> "3"
+  it should "execute `1 + 2` == 3" in withSession { session =>
+    val statement = session.execute("1 + 2")
+    statement.id should equal (0)
+
+    val result = Await.result(statement.result, Duration.Inf)
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> "3"
+      )
+    ))
+
+    result should equal (expectedResult)
+  }
+
+  it should "execute `x = 1`, then `y = 2`, then `x + y`" in withSession { session =>
+    var statement = session.execute("x = 1")
+    statement.id should equal (0)
+
+    var result = Await.result(statement.result, Duration.Inf)
+    var expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> ""
+      )
+    ))
+
+    result should equal (expectedResult)
+
+    statement = session.execute("y = 2")
+    statement.id should equal (1)
+
+    result = Await.result(statement.result, Duration.Inf)
+    expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 1,
+      "data" -> Map(
+        "text/plain" -> ""
+      )
+    ))
+
+    result should equal (expectedResult)
+
+    statement = session.execute("x + y")
+    statement.id should equal (2)
+
+    result = Await.result(statement.result, Duration.Inf)
+    expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 2,
+      "data" -> Map(
+        "text/plain" -> "3"
+      )
+    ))
+
+    result should equal (expectedResult)
+  }
+
+  it should "do table magic" in withSession { session =>
+    val statement = session.execute("x = [[1, 'a'], [3, 'b']]\n%table x")
+    statement.id should equal (0)
+
+    val result = Await.result(statement.result, Duration.Inf)
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "application/vnd.livy.table.v1+json" -> Map(
+          "headers" -> List(
+            Map("type" -> "INT_TYPE", "name" -> "0"),
+            Map("type" -> "STRING_TYPE", "name" -> "1")),
+          "data" -> List(List(1, "a"), List(3, "b"))
         )
-      ))
+      )
+    ))
 
-      result should equal (expectedResult)
-    }
+    result should equal (expectedResult)
+  }
 
-    it("should execute `x = 1`, then `y = 2`, then `x + y`") {
-      var statement = session.execute("x = 1")
-      statement.id should equal (0)
+  it should "capture stdout" in withSession { session =>
+    val statement = session.execute("""print 'Hello World'""")
+    statement.id should equal (0)
 
-      var result = Await.result(statement.result, Duration.Inf)
-      var expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> ""
-        )
-      ))
+    val result = Await.result(statement.result, Duration.Inf)
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> "Hello World"
+      )
+    ))
+
+    result should equal (expectedResult)
+  }
 
-      result should equal (expectedResult)
+  it should "report an error if accessing an unknown variable" in withSession { session =>
+    val statement = session.execute("""x""")
+    statement.id should equal (0)
+
+    val result = Await.result(statement.result, Duration.Inf)
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "error",
+      "execution_count" -> 0,
+      "traceback" -> List(
+        "Traceback (most recent call last):\n",
+        "NameError: name 'x' is not defined\n"
+      ),
+      "ename" -> "NameError",
+      "evalue" -> "name 'x' is not defined"
+    ))
+
+    result should equal (expectedResult)
+  }
 
-      statement = session.execute("y = 2")
-      statement.id should equal (1)
+  it should "report an error if exception is thrown" in withSession { session =>
+    val statement = session.execute(
+      """def foo():
+        |    raise Exception()
+        |foo()
+        |""".stripMargin)
+    statement.id should equal (0)
+
+    val result = Await.result(statement.result, Duration.Inf)
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "error",
+      "execution_count" -> 0,
+      "traceback" -> List(
+        "Traceback (most recent call last):\n",
+        "Exception\n"
+      ),
+      "ename" -> "Exception",
+      "evalue" -> ""
+    ))
+
+    result should equal (expectedResult)
+  }
 
-      result = Await.result(statement.result, Duration.Inf)
-      expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 1,
-        "data" -> Map(
-          "text/plain" -> ""
-        )
-      ))
+  it should "access the spark context" in withSession { session =>
+    val statement = session.execute("""sc""")
+    statement.id should equal (0)
 
-      result should equal (expectedResult)
+    val result = Await.result(statement.result, Duration.Inf)
+    val resultMap = result.extract[Map[String, JValue]]
 
-      statement = session.execute("x + y")
-      statement.id should equal (2)
+    // Manually extract the values since the line numbers in the exception could change.
+    resultMap("status").extract[String] should equal ("ok")
+    resultMap("execution_count").extract[Int] should equal (0)
 
-      result = Await.result(statement.result, Duration.Inf)
-      expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 2,
-        "data" -> Map(
-          "text/plain" -> "3"
-        )
-      ))
-
-      result should equal (expectedResult)
-    }
-
-    it("should do table magic") {
-      val statement = session.execute("x = [[1, 'a'], [3, 'b']]\n%table x")
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "application/vnd.livy.table.v1+json" -> Map(
-            "headers" -> List(
-              Map("type" -> "INT_TYPE", "name" -> "0"),
-              Map("type" -> "STRING_TYPE", "name" -> "1")),
-            "data" -> List(List(1, "a"), List(3, "b"))
-          )
-        )
-      ))
+    val data = resultMap("data").extract[Map[String, JValue]]
+    data("text/plain").extract[String] should include ("<pyspark.context.SparkContext object at")
+  }
 
-      result should equal (expectedResult)
-    }
+  it should "execute spark commands" in withSession { session =>
+    val statement = session.execute("""
+                                      |sc.parallelize(xrange(0, 2)).map(lambda i: i + 1).collect()
+                                      |""".stripMargin)
+    statement.id should equal (0)
 
-    it("should capture stdout") {
-      val statement = session.execute("""print 'Hello World'""")
-      statement.id should equal (0)
+    val result = Await.result(statement.result, Duration.Inf)
 
-      val result = Await.result(statement.result, Duration.Inf)
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> "Hello World"
-        )
-      ))
-
-      result should equal (expectedResult)
-    }
-
-    it("should report an error if accessing an unknown variable") {
-      val statement = session.execute("""x""")
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "error",
-        "execution_count" -> 0,
-        "traceback" -> List(
-          "Traceback (most recent call last):\n",
-          "NameError: name 'x' is not defined\n"
-        ),
-        "ename" -> "NameError",
-        "evalue" -> "name 'x' is not defined"
-      ))
-
-      result should equal (expectedResult)
-    }
-
-    it("should report an error if exception is thrown") {
-      val statement = session.execute(
-        """def foo():
-          |    raise Exception()
-          |foo()
-          |""".stripMargin)
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "error",
-        "execution_count" -> 0,
-        "traceback" -> List(
-          "Traceback (most recent call last):\n",
-          "Exception\n"
-        ),
-        "ename" -> "Exception",
-        "evalue" -> ""
-      ))
-
-      result should equal (expectedResult)
-    }
-
-    it("should access the spark context") {
-      val statement = session.execute("""sc""")
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val resultMap = result.extract[Map[String, JValue]]
-
-      // Manually extract the values since the line numbers in the exception could change.
-      resultMap("status").extract[String] should equal ("ok")
-      resultMap("execution_count").extract[Int] should equal (0)
-
-      val data = resultMap("data").extract[Map[String, JValue]]
-      data("text/plain").extract[String] should include ("<pyspark.context.SparkContext object at")
-    }
-
-    it("should execute spark commands") {
-      val statement = session.execute("""
-          |sc.parallelize(xrange(0, 2)).map(lambda i: i + 1).collect()
-          |""".stripMargin)
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> "[1, 2]"
-        )
-      ))
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> "[1, 2]"
+      )
+    ))
 
-      result should equal (expectedResult)
-    }
+    result should equal (expectedResult)
   }
 }

+ 95 - 97
apps/spark/java/livy-repl/src/test/scala/com/cloudera/hue/livy/repl/ScalaInterpreterSpec.scala

@@ -29,102 +29,100 @@ class ScalaInterpreterSpec extends BaseInterpreterSpec {
 
   override def createInterpreter() = SparkInterpreter()
 
-  describe("A spark interpreter") {
-    it("should execute `1 + 2` == 3") {
-      val response = interpreter.execute("1 + 2")
-      response should equal (Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "res0: Int = 3"
-      ))
-    }
-
-    it("should execute multiple statements") {
-      var response = interpreter.execute("val x = 1")
-      response should equal (Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "x: Int = 1"
-      ))
-
-      response = interpreter.execute("val y = 2")
-      response should equal (Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "y: Int = 2"
-      ))
-
-      response = interpreter.execute("x + y")
-      response should equal (Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "res0: Int = 3"
-      ))
-    }
-
-    it("should execute multiple statements in one block") {
-      val response = interpreter.execute(
-        """
-          |val x = 1
-          |
-          |val y = 2
-          |
-          |x + y
-        """.stripMargin)
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "res2: Int = 3"
-      ))
-    }
-
-    it("should do table magic") {
-      val response = interpreter.execute(
-        """val x = List(List(1, "a"), List(3, "b"))
-          |%table x
-        """.stripMargin)
-
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.APPLICATION_LIVY_TABLE_JSON -> (
-          ("headers" -> List(
-            ("type" -> "BIGINT_TYPE") ~ ("name" -> "0"),
-            ("type" -> "STRING_TYPE") ~ ("name" -> "1")
-          )) ~
-            ("data" -> List(
-              List[JValue](1, "a"),
-              List[JValue](3, "b")
-            ))
-          )
-      ))
-    }
-
-    it("should allow magic inside statements") {
-      val response = interpreter.execute(
-        """val x = List(List(1, "a"), List(3, "b"))
-          |%table x
-          |1 + 2
-        """.stripMargin)
-
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "res0: Int = 3"
-      ))
-    }
-
-    it("should capture stdout") {
-      val response = interpreter.execute("println(\"Hello World\")")
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "Hello World"
-      ))
-    }
-
-    it("should report an error if accessing an unknown variable") {
-      val response = interpreter.execute("x")
-      response should equal(Interpreter.ExecuteError(
-        "Error",
-        """<console>:8: error: not found: value x
-          |              x
-          |              ^""".stripMargin,
-        List()
-      ))
-    }
-
-    it("should execute spark commands") {
-      val response = interpreter.execute(
-        """sc.parallelize(0 to 1).map { i => i+1 }.collect""".stripMargin)
-
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "res0: Array[Int] = Array(1, 2)"
-      ))
-    }
+  it should "execute `1 + 2` == 3" in withInterpreter { interpreter =>
+    val response = interpreter.execute("1 + 2")
+    response should equal (Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "res0: Int = 3"
+    ))
+  }
+
+  it should "execute multiple statements" in withInterpreter { interpreter =>
+    var response = interpreter.execute("val x = 1")
+    response should equal (Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "x: Int = 1"
+    ))
+
+    response = interpreter.execute("val y = 2")
+    response should equal (Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "y: Int = 2"
+    ))
+
+    response = interpreter.execute("x + y")
+    response should equal (Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "res0: Int = 3"
+    ))
+  }
+
+  it should "execute multiple statements in one block" in withInterpreter { interpreter =>
+    val response = interpreter.execute(
+      """
+        |val x = 1
+        |
+        |val y = 2
+        |
+        |x + y
+      """.stripMargin)
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "res2: Int = 3"
+    ))
+  }
+
+  it should "do table magic" in withInterpreter { interpreter =>
+    val response = interpreter.execute(
+      """val x = List(List(1, "a"), List(3, "b"))
+        |%table x
+      """.stripMargin)
+
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.APPLICATION_LIVY_TABLE_JSON -> (
+        ("headers" -> List(
+          ("type" -> "BIGINT_TYPE") ~ ("name" -> "0"),
+          ("type" -> "STRING_TYPE") ~ ("name" -> "1")
+        )) ~
+          ("data" -> List(
+            List[JValue](1, "a"),
+            List[JValue](3, "b")
+          ))
+        )
+    ))
+  }
+
+  it should "allow magic inside statements" in withInterpreter { interpreter =>
+    val response = interpreter.execute(
+      """val x = List(List(1, "a"), List(3, "b"))
+        |%table x
+        |1 + 2
+      """.stripMargin)
+
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "res0: Int = 3"
+    ))
+  }
+
+  it should "capture stdout" in withInterpreter { interpreter =>
+    val response = interpreter.execute("println(\"Hello World\")")
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "Hello World"
+    ))
+  }
+
+  it should "report an error if accessing an unknown variable" in withInterpreter { interpreter =>
+    val response = interpreter.execute("x")
+    response should equal(Interpreter.ExecuteError(
+      "Error",
+      """<console>:8: error: not found: value x
+        |              x
+        |              ^""".stripMargin,
+      List()
+    ))
+  }
+
+  it should "execute spark commands" in withInterpreter { interpreter =>
+    val response = interpreter.execute(
+      """sc.parallelize(0 to 1).map { i => i+1 }.collect""".stripMargin)
+
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "res0: Array[Int] = Array(1, 2)"
+    ))
   }
 }

+ 73 - 67
apps/spark/java/livy-repl/src/test/scala/com/cloudera/hue/livy/repl/SparkRInterpreterSpec.scala

@@ -21,85 +21,91 @@ package com.cloudera.hue.livy.repl
 import com.cloudera.hue.livy.repl
 import com.cloudera.hue.livy.repl.sparkr.SparkRInterpreter
 import org.json4s.JsonDSL._
-import org.json4s.{JObject, DefaultFormats, JValue}
+import org.json4s.{DefaultFormats, JValue}
 
 class SparkRInterpreterSpec extends BaseInterpreterSpec {
 
   implicit val formats = DefaultFormats
 
-  override def createInterpreter() = SparkRInterpreter()
+  override protected def withFixture(test: NoArgTest) = {
+    val sparkRExecutable = SparkRInterpreter.sparkRExecutable
+    assume(sparkRExecutable.isDefined, "Cannot find sparkR")
+    test()
+  }
 
-  describe("A python interpreter") {
-    it("should execute `1 + 2` == 3") {
-      val response = interpreter.execute("1 + 2")
-      response should equal (Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "[1] 3"
-      ))
-    }
+  override def createInterpreter() = {
+    SparkRInterpreter()
+  }
 
-    it("should execute multiple statements") {
-      var response = interpreter.execute("x = 1")
-      response should equal (Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> ""
-      ))
-
-      response = interpreter.execute("y = 2")
-      response should equal (Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> ""
-      ))
-
-      response = interpreter.execute("x + y")
-      response should equal (Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "[1] 3"
-      ))
-    }
+  it should "execute `1 + 2` == 3" in withInterpreter { interpreter =>
+    val response = interpreter.execute("1 + 2")
+    response should equal (Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "[1] 3"
+    ))
+  }
 
-    it("should execute multiple statements in one block") {
-      val response = interpreter.execute(
-        """
-          |x = 1
-          |
-          |y = 2
-          |
-          |x + y
-        """.stripMargin)
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "[1] 3"
-      ))
-    }
+  it should "execute multiple statements" in withInterpreter { interpreter =>
+    var response = interpreter.execute("x = 1")
+    response should equal (Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> ""
+    ))
 
-    it("should capture stdout") {
-      val response = interpreter.execute("cat(3)")
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "3"
-      ))
-    }
+    response = interpreter.execute("y = 2")
+    response should equal (Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> ""
+    ))
 
-    it("should report an error if accessing an unknown variable") {
-      val response = interpreter.execute("x")
-      response should equal(Interpreter.ExecuteSuccess(
-        repl.TEXT_PLAIN -> "Error: object 'x' not found"
-      ))
-    }
+    response = interpreter.execute("x + y")
+    response should equal (Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "[1] 3"
+    ))
+  }
+
+  it should "execute multiple statements in one block" in withInterpreter { interpreter =>
+    val response = interpreter.execute(
+      """
+        |x = 1
+        |
+        |y = 2
+        |
+        |x + y
+      """.stripMargin)
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "[1] 3"
+    ))
+  }
+
+  it should "capture stdout" in withInterpreter { interpreter =>
+    val response = interpreter.execute("cat(3)")
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "3"
+    ))
+  }
+
+  it should "report an error if accessing an unknown variable" in withInterpreter { interpreter =>
+    val response = interpreter.execute("x")
+    response should equal(Interpreter.ExecuteSuccess(
+      repl.TEXT_PLAIN -> "Error: object 'x' not found"
+    ))
+  }
 
-    it("should execute spark commands") {
-      val response = interpreter.execute(
-        """head(createDataFrame(sqlContext, faithful))""")
-
-      response match {
-        case Interpreter.ExecuteSuccess(map: JValue) =>
-          (map \ "text/plain").extract[String] should include (
-            """  eruptions waiting
-              |1     3.600      79
-              |2     1.800      54
-              |3     3.333      74
-              |4     2.283      62
-              |5     4.533      85
-              |6     2.883      55""".stripMargin)
-        case _ =>
-          throw new Exception("response is not a success")
-      }
+  it should "execute spark commands" in withInterpreter { interpreter =>
+    val response = interpreter.execute(
+      """head(createDataFrame(sqlContext, faithful))""")
 
+    response match {
+      case Interpreter.ExecuteSuccess(map: JValue) =>
+        (map \ "text/plain").extract[String] should include (
+          """  eruptions waiting
+            |1     3.600      79
+            |2     1.800      54
+            |3     3.333      74
+            |4     2.283      62
+            |5     4.533      85
+            |6     2.883      55""".stripMargin)
+      case _ =>
+        throw new Exception("response is not a success")
     }
+
   }
 }

+ 146 - 141
apps/spark/java/livy-repl/src/test/scala/com/cloudera/hue/livy/repl/SparkRSessionSpec.scala

@@ -21,160 +21,165 @@ package com.cloudera.hue.livy.repl
 import com.cloudera.hue.livy.repl.sparkr.SparkRInterpreter
 import org.json4s.Extraction
 import org.json4s.JsonAST.JValue
+import org.scalatest.BeforeAndAfterAll
 
 import _root_.scala.concurrent.Await
 import _root_.scala.concurrent.duration.Duration
 
 class SparkRSessionSpec extends BaseSessionSpec {
 
+  override protected def withFixture(test: NoArgTest) = {
+    val sparkRExecutable = SparkRInterpreter.sparkRExecutable
+    assume(sparkRExecutable.isDefined, "Cannot find sparkR")
+    test()
+  }
+
   override def createInterpreter() = SparkRInterpreter()
 
-  describe("A sparkr session") {
-    it("should execute `1 + 2` == 3") {
-      val statement = session.execute("1 + 2")
-      statement.id should equal(0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> "[1] 3"
-        )
-      ))
-
-      result should equal(expectedResult)
-    }
-
-    it("should execute `x = 1`, then `y = 2`, then `x + y`") {
-      var statement = session.execute("x = 1")
-      statement.id should equal (0)
-
-      var result = Await.result(statement.result, Duration.Inf)
-      var expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> ""
-        )
-      ))
-
-      result should equal (expectedResult)
-
-      statement = session.execute("y = 2")
-      statement.id should equal (1)
-
-      result = Await.result(statement.result, Duration.Inf)
-      expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 1,
-        "data" -> Map(
-          "text/plain" -> ""
-        )
-      ))
-
-      result should equal (expectedResult)
-
-      statement = session.execute("x + y")
-      statement.id should equal (2)
-
-      result = Await.result(statement.result, Duration.Inf)
-      expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 2,
-        "data" -> Map(
-          "text/plain" -> "[1] 3"
-        )
-      ))
-
-      result should equal (expectedResult)
-    }
-
-    it("should capture stdout from print") {
-      val statement = session.execute("""print('Hello World')""")
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> "[1] \"Hello World\""
-        )
-      ))
-
-      result should equal (expectedResult)
-    }
-
-    it("should capture stdout from cat") {
-      val statement = session.execute("""cat(3)""")
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> "3"
-        )
-      ))
-
-      result should equal (expectedResult)
-    }
-
-    it("should report an error if accessing an unknown variable") {
-      val statement = session.execute("""x""")
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> "Error: object 'x' not found"
-        )
-      ))
-
-      result should equal (expectedResult)
-    }
-
-    it("should access the spark context") {
-      val statement = session.execute("""sc""")
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val resultMap = result.extract[Map[String, JValue]]
-
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> "Java ref type org.apache.spark.api.java.JavaSparkContext id 0"
-        )
-      ))
-    }
-
-    it("should execute spark commands") {
-      val statement = session.execute("""
-                                        |head(createDataFrame(sqlContext, faithful))
-                                        |""".stripMargin)
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val resultMap = result.extract[Map[String, JValue]]
-
-      // Manually extract since sparkr outputs a lot of spark logging information.
-      resultMap("status").extract[String] should equal ("ok")
-      resultMap("execution_count").extract[Int] should equal (0)
-
-      val data = resultMap("data").extract[Map[String, JValue]]
-      data("text/plain").extract[String] should include ("""  eruptions waiting
+  it should "execute `1 + 2` == 3" in withSession { session =>
+    val statement = session.execute("1 + 2")
+    statement.id should equal(0)
+
+    val result = Await.result(statement.result, Duration.Inf)
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> "[1] 3"
+      )
+    ))
+
+    result should equal(expectedResult)
+  }
+
+    it should "execute `x = 1`, then `y = 2`, then `x + y`" in withSession { session =>
+    var statement = session.execute("x = 1")
+    statement.id should equal (0)
+
+    var result = Await.result(statement.result, Duration.Inf)
+    var expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> ""
+      )
+    ))
+
+    result should equal (expectedResult)
+
+    statement = session.execute("y = 2")
+    statement.id should equal (1)
+
+    result = Await.result(statement.result, Duration.Inf)
+    expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 1,
+      "data" -> Map(
+        "text/plain" -> ""
+      )
+    ))
+
+    result should equal (expectedResult)
+
+    statement = session.execute("x + y")
+    statement.id should equal (2)
+
+    result = Await.result(statement.result, Duration.Inf)
+    expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 2,
+      "data" -> Map(
+        "text/plain" -> "[1] 3"
+      )
+    ))
+
+    result should equal (expectedResult)
+  }
+
+    it should "capture stdout from print" in withSession { session =>
+    val statement = session.execute("""print('Hello World')""")
+    statement.id should equal (0)
+
+    val result = Await.result(statement.result, Duration.Inf)
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> "[1] \"Hello World\""
+      )
+    ))
+
+    result should equal (expectedResult)
+  }
+
+    it should "capture stdout from cat" in withSession { session =>
+    val statement = session.execute("""cat(3)""")
+    statement.id should equal (0)
+
+    val result = Await.result(statement.result, Duration.Inf)
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> "3"
+      )
+    ))
+
+    result should equal (expectedResult)
+  }
+
+    it should "report an error if accessing an unknown variable" in withSession { session =>
+    val statement = session.execute("""x""")
+    statement.id should equal (0)
+
+    val result = Await.result(statement.result, Duration.Inf)
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> "Error: object 'x' not found"
+      )
+    ))
+
+    result should equal (expectedResult)
+  }
+
+    it should "access the spark context" in withSession { session =>
+    val statement = session.execute("""sc""")
+    statement.id should equal (0)
+
+    val result = Await.result(statement.result, Duration.Inf)
+    val resultMap = result.extract[Map[String, JValue]]
+
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> "Java ref type org.apache.spark.api.java.JavaSparkContext id 0"
+      )
+    ))
+  }
+
+    it should "execute spark commands" in withSession { session =>
+    val statement = session.execute("""
+                                      |head(createDataFrame(sqlContext, faithful))
+                                      |""".stripMargin)
+    statement.id should equal (0)
+
+    val result = Await.result(statement.result, Duration.Inf)
+    val resultMap = result.extract[Map[String, JValue]]
+
+    // Manually extract since sparkr outputs a lot of spark logging information.
+    resultMap("status").extract[String] should equal ("ok")
+    resultMap("execution_count").extract[Int] should equal (0)
+
+    val data = resultMap("data").extract[Map[String, JValue]]
+    data("text/plain").extract[String] should include ("""  eruptions waiting
                                                          |1     3.600      79
                                                          |2     1.800      54
                                                          |3     3.333      74
                                                          |4     2.283      62
                                                          |5     4.533      85
                                                          |6     2.883      55""".stripMargin)
-    }
   }
 }

+ 146 - 148
apps/spark/java/livy-repl/src/test/scala/com/cloudera/hue/livy/repl/SparkSessionSpec.scala

@@ -29,171 +29,169 @@ class SparkSessionSpec extends BaseSessionSpec {
 
   override def createInterpreter() = SparkInterpreter()
 
-  describe("A spark session") {
-    it("should execute `1 + 2` == 3") {
-      val statement = session.execute("1 + 2")
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> "res0: Int = 3"
-        )
-      ))
+  it should "execute `1 + 2` == 3" in withSession { session =>
+    val statement = session.execute("1 + 2")
+    statement.id should equal (0)
+
+    val result = Await.result(statement.result, Duration.Inf)
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> "res0: Int = 3"
+      )
+    ))
+
+    result should equal (expectedResult)
+  }
 
-      result should equal (expectedResult)
-    }
+  it should "execute `x = 1`, then `y = 2`, then `x + y`" in withSession { session =>
+    var statement = session.execute("val x = 1")
+    statement.id should equal (0)
+
+    var result = Await.result(statement.result, Duration.Inf)
+    var expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> "x: Int = 1"
+      )
+    ))
+
+    result should equal (expectedResult)
+
+    statement = session.execute("val y = 2")
+    statement.id should equal (1)
+
+    result = Await.result(statement.result, Duration.Inf)
+    expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 1,
+      "data" -> Map(
+        "text/plain" -> "y: Int = 2"
+      )
+    ))
+
+    result should equal (expectedResult)
+
+    statement = session.execute("x + y")
+    statement.id should equal (2)
+
+    result = Await.result(statement.result, Duration.Inf)
+    expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 2,
+      "data" -> Map(
+        "text/plain" -> "res0: Int = 3"
+      )
+    ))
+
+    result should equal (expectedResult)
+  }
 
-    it("should execute `x = 1`, then `y = 2`, then `x + y`") {
-      var statement = session.execute("val x = 1")
-      statement.id should equal (0)
+  it should "capture stdout" in withSession { session =>
+    val statement = session.execute("""println("Hello World")""")
+    statement.id should equal (0)
 
-      var result = Await.result(statement.result, Duration.Inf)
-      var expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> "x: Int = 1"
-        )
-      ))
+    val result = Await.result(statement.result, Duration.Inf)
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> "Hello World"
+      )
+    ))
 
-      result should equal (expectedResult)
+    result should equal (expectedResult)
+  }
+
+  it should "report an error if accessing an unknown variable" in withSession { session =>
+    val statement = session.execute("""x""")
+    statement.id should equal (0)
+
+    val result = Await.result(statement.result, Duration.Inf)
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "error",
+      "execution_count" -> 0,
+      "ename" -> "Error",
+      "evalue" ->
+        """<console>:8: error: not found: value x
+          |              x
+          |              ^""".stripMargin,
+      "traceback" -> List()
+    ))
+
+    result should equal (expectedResult)
+  }
 
-      statement = session.execute("val y = 2")
-      statement.id should equal (1)
+  it should "report an error if exception is thrown" in withSession { session =>
+    val statement = session.execute("""throw new Exception()""")
+    statement.id should equal (0)
 
-      result = Await.result(statement.result, Duration.Inf)
-      expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 1,
-        "data" -> Map(
-          "text/plain" -> "y: Int = 2"
-        )
-      ))
+    val result = Await.result(statement.result, Duration.Inf)
+    val resultMap = result.extract[Map[String, JValue]]
 
-      result should equal (expectedResult)
+    // Manually extract the values since the line numbers in the exception could change.
+    resultMap("status").extract[String] should equal ("error")
+    resultMap("execution_count").extract[Int] should equal (0)
+    resultMap("ename").extract[String] should equal ("Error")
+    resultMap("evalue").extract[String] should include ("java.lang.Exception")
+    resultMap("traceback").extract[List[_]] should equal (List())
+  }
 
-      statement = session.execute("x + y")
-      statement.id should equal (2)
+  it should "access the spark context" in withSession { session =>
+    val statement = session.execute("""sc""")
+    statement.id should equal (0)
 
-      result = Await.result(statement.result, Duration.Inf)
-      expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 2,
-        "data" -> Map(
-          "text/plain" -> "res0: Int = 3"
-        )
-      ))
+    val result = Await.result(statement.result, Duration.Inf)
+    val resultMap = result.extract[Map[String, JValue]]
 
-      result should equal (expectedResult)
-    }
+    // Manually extract the values since the line numbers in the exception could change.
+    resultMap("status").extract[String] should equal ("ok")
+    resultMap("execution_count").extract[Int] should equal (0)
 
-    it("should capture stdout") {
-      val statement = session.execute("""println("Hello World")""")
-      statement.id should equal (0)
+    val data = resultMap("data").extract[Map[String, JValue]]
+    data("text/plain").extract[String] should include ("res0: org.apache.spark.SparkContext = org.apache.spark.SparkContext")
+  }
 
-      val result = Await.result(statement.result, Duration.Inf)
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> "Hello World"
-        )
-      ))
-
-      result should equal (expectedResult)
-    }
-
-    it("should report an error if accessing an unknown variable") {
-      val statement = session.execute("""x""")
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "error",
-        "execution_count" -> 0,
-        "ename" -> "Error",
-        "evalue" ->
-          """<console>:8: error: not found: value x
-            |              x
-            |              ^""".stripMargin,
-        "traceback" -> List()
-      ))
-
-      result should equal (expectedResult)
-    }
-
-    it("should report an error if exception is thrown") {
-      val statement = session.execute("""throw new Exception()""")
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val resultMap = result.extract[Map[String, JValue]]
-
-      // Manually extract the values since the line numbers in the exception could change.
-      resultMap("status").extract[String] should equal ("error")
-      resultMap("execution_count").extract[Int] should equal (0)
-      resultMap("ename").extract[String] should equal ("Error")
-      resultMap("evalue").extract[String] should include ("java.lang.Exception")
-      resultMap("traceback").extract[List[_]] should equal (List())
-    }
-
-    it("should access the spark context") {
-      val statement = session.execute("""sc""")
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-      val resultMap = result.extract[Map[String, JValue]]
-
-      // Manually extract the values since the line numbers in the exception could change.
-      resultMap("status").extract[String] should equal ("ok")
-      resultMap("execution_count").extract[Int] should equal (0)
-
-      val data = resultMap("data").extract[Map[String, JValue]]
-      data("text/plain").extract[String] should include ("res0: org.apache.spark.SparkContext = org.apache.spark.SparkContext")
-    }
-
-    it("should execute spark commands") {
-      val statement = session.execute(
-        """sc.parallelize(0 to 1).map{i => i+1}.collect""".stripMargin)
-      statement.id should equal (0)
-
-      val result = Await.result(statement.result, Duration.Inf)
-
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "text/plain" -> "res0: Array[Int] = Array(1, 2)"
-        )
-      ))
+  it should "execute spark commands" in withSession { session =>
+    val statement = session.execute(
+      """sc.parallelize(0 to 1).map{i => i+1}.collect""".stripMargin)
+    statement.id should equal (0)
+
+    val result = Await.result(statement.result, Duration.Inf)
 
-      result should equal (expectedResult)
-    }
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "text/plain" -> "res0: Array[Int] = Array(1, 2)"
+      )
+    ))
+
+    result should equal (expectedResult)
+  }
 
-    it("should do table magic") {
-      val statement = session.execute("val x = List((1, \"a\"), (3, \"b\"))\n%table x")
-      statement.id should equal (0)
+  it should "do table magic" in withSession { session =>
+    val statement = session.execute("val x = List((1, \"a\"), (3, \"b\"))\n%table x")
+    statement.id should equal (0)
 
-      val result = Await.result(statement.result, Duration.Inf)
+    val result = Await.result(statement.result, Duration.Inf)
 
 
-      val expectedResult = Extraction.decompose(Map(
-        "status" -> "ok",
-        "execution_count" -> 0,
-        "data" -> Map(
-          "application/vnd.livy.table.v1+json" -> Map(
-            "headers" -> List(
-              Map("type" -> "BIGINT_TYPE", "name" -> "_1"),
-              Map("type" -> "STRING_TYPE", "name" -> "_2")),
-            "data" -> List(List(1, "a"), List(3, "b"))
-          )
+    val expectedResult = Extraction.decompose(Map(
+      "status" -> "ok",
+      "execution_count" -> 0,
+      "data" -> Map(
+        "application/vnd.livy.table.v1+json" -> Map(
+          "headers" -> List(
+            Map("type" -> "BIGINT_TYPE", "name" -> "_1"),
+            Map("type" -> "STRING_TYPE", "name" -> "_2")),
+          "data" -> List(List(1, "a"), List(3, "b"))
         )
-      ))
+      )
+    ))
 
-      result should equal (expectedResult)
-    }
+    result should equal (expectedResult)
   }
- }
+}

+ 1 - 1
apps/spark/java/livy-server/src/main/scala/com/cloudera/hue/livy/server/Main.scala

@@ -137,7 +137,7 @@ object Main {
 
     output match {
       case regex(version) => version
-      case _ => throw new IOException(f"Unable to determing spark-submit version [$exitCode]:\n$output")
+      case _ => throw new IOException(f"Unable to determine spark-submit version [$exitCode]:\n$output")
     }
   }