Browse Source

HUE-5569 [spark] Add "Spark Conf" param to Spark session config dropdown

It allows the user to submit key-value pairs of Spark configuration that will be added to the properties file submitted with the session create.
Rick Bernotas 9 years ago
parent
commit
4c5970d77d
1 changed files with 21 additions and 0 deletions
  1. 21 0
      desktop/libs/notebook/src/notebook/connectors/spark_shell.py

+ 21 - 0
desktop/libs/notebook/src/notebook/connectors/spark_shell.py

@@ -46,6 +46,16 @@ class SparkConfiguration(object):
   APP_NAME = 'spark'
 
   PROPERTIES = [
+    {
+      "name": "conf",
+      "nice_name": _("Spark Conf"),
+      "help_text": _("Add one or more Spark conf properties to the session."),
+      "type": "settings",
+      "is_yarn": False,
+      "multiple": True,
+      "defaultValue": [],
+      "value": [],
+    },
     {
       "name": "jars",
       "nice_name": _("Jars"),
@@ -186,6 +196,17 @@ class SparkApi(Api):
       props['pyFiles'] = tmppyFiles.split(",")
       LOG.debug("Check List type: pyFiles was not a list")
 
+    # Convert the conf list to a dict for Livy
+    listitems = props['conf']
+    LOG.debug("Property Spark Conf kvp list from UI is: " + str(listitems))
+    confDict = {}
+    for i in range(len(listitems)):
+       kvp = listitems[i]
+       LOG.debug("Property Spark Conf key " + str(i) + " = " + str(kvp.get('key')))
+       LOG.debug("Property Spark Conf value " + str(i) + " = " + str(kvp.get('value')))
+       confDict[kvp.get('key')] = kvp.get('value')
+    props['conf'] = confDict
+    LOG.debug("Property Spark Conf dictionary is: " + str(confDict))
 
     props['kind'] = lang