瀏覽代碼

[spark] Make session properties generic

Romain Rigaux 10 年之前
父節點
當前提交
8ec5cca

+ 17 - 3
apps/spark/src/spark/models.py

@@ -20,6 +20,8 @@ import logging
 import re
 import time
 
+from django.utils.translation import ugettext as _
+
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.i18n import force_unicode
 
@@ -285,10 +287,22 @@ class HS2Api(Api):
 # Spark
 
 class SparkApi(Api):
+  PROPERTIES = [
+    {'name': 'jars', 'nice_name': _('Jars'), 'default': '', 'type': 'csv', 'is_yarn': False},
+    {'name': 'files', 'nice_name': _('Files'), 'default': '', 'type': 'csv', 'is_yarn': False},
+    {'name': 'pyFiles', 'nice_name': _('pyFiles'), 'default': '', 'type': 'csv', 'is_yarn': False},
+    
+    {'name': 'driverMemory', 'nice_name': _('Driver Memory'), 'default': '1', 'type': 'jvm', 'is_yarn': False},
+    
+    {'name': 'driverCores', 'nice_name': _('Driver Cores'), 'default': '1', 'type': 'number', 'is_yarn': True},
+    {'name': 'executorCores', 'nice_name': _('Executor Cores'), 'default': '1', 'type': 'number', 'is_yarn': True},
+    {'name': 'queue', 'nice_name': _('Queue'), 'default': '1', 'type': 'string', 'is_yarn': True},
+    {'name': 'archives', 'nice_name': _('Archives'), 'default': '', 'type': 'csv', 'is_yarn': True},
+    {'name': 'numExecutors', 'nice_name': _('Executors Numbers'), 'default': '1', 'type': 'number', 'is_yarn': True},    
+  ]
 
   def create_session(self, lang='scala', properties=None):
-    if properties is None:
-      properties = {'executorMemory': '500M'}
+    properties = dict([(p['name'], p['value']) for p in properties]) if properties is not None else {}
 
     properties['kind'] = lang
 
@@ -310,7 +324,7 @@ class SparkApi(Api):
     return {
         'type': lang,
         'id': response['id'],
-        'properties': properties
+        'properties': []
     }
 
   def execute(self, notebook, snippet):

+ 17 - 1
apps/spark/src/spark/static/spark/js/spark.ko.js

@@ -797,12 +797,28 @@ function EditorViewModel(notebooks, options) {
     $.totalStorage('spark_assist_visible', newValue);
   });
 
-
   self.assistContent = ko.observable();
   self.assistSelectedMainObject = ko.observable();
 
   self.availableSnippets = ko.mapping.fromJS(options.languages);
   self.snippetPlaceholders = options.snippet_placeholders;
+ 
+  self.availableSessionProperties = ko.computed(function () { // Only Spark
+    return ko.utils.arrayFilter(options.session_properties, function (item) {
+        return item.name != '' // Could filter out the ones already selected + yarn only or not
+      });
+  });
+  self.selectedSessionProperties = ko.observable();
+  self.getSessionProperties = function(name) {
+    var _prop = null;
+    $.each(options.session_properties, function(index, prop) {
+      if (prop.name == name) {
+        _prop = prop;
+        return;
+      }
+    });console.log(_prop);
+    return _prop;
+  };
 
   self.getSnippetName = function(snippetType)  {
     var availableSnippets = self.availableSnippets();

+ 33 - 39
apps/spark/src/spark/templates/editor_components.mako

@@ -777,48 +777,43 @@ from django.utils.translation import ugettext as _
             <p>${ _('There are currently no active sessions.') }</p>
             <!-- /ko -->
             <!-- ko foreach: sessions -->
-            <!-- ko with: $data.properties -->
-            <h4 data-bind="text: $root.getSnippetName($parent.type())" style="clear:left;"></h4>
-            <!-- ko if: typeof executor_cores !== 'undefined' -->
-            <div class="control-group" style="float: left;">
-              <label class="control-label">${_('Executor Cores')}</label>
-              <div class="controls">
-                <input class="input-small" type="text" data-bind="numericTextInput: { value: executor_cores, precision: 0 }" />
-              </div>
-            </div>
-            <!-- /ko -->
-            <!-- ko if: typeof executor_memory !== 'undefined' -->
-            <div class="control-group" style="float: left;">
-              <label class="control-label">${_('Executor Memory')}</label>
-              <div class="controls">
-                <jvm-memory-input params="value: executor_memory" />
-              </div>
-            </div>
-            <!-- /ko -->
-            <!-- ko if: typeof executor_count !== 'undefined' -->
-            <div class="control-group" style="float: left;">
-              <label class="control-label">${_('Executor Count')}</label>
-              <div class="controls">
-                <input class="input-small" type="text" data-bind="numericTextInput: { value: executor_count, precision: 0 }" />
-              </div>
-            </div>
-            <!-- /ko -->
-            <!-- ko if: typeof driver_cores !== 'undefined' -->
-            <div class="control-group cl" style="float: left; clear: left;">
-              <label class="control-label">${_('Driver Cores')}</label>
-              <div class="controls">
-                <input class="input-small" type="text" data-bind="numericTextInput: { value: driver_cores, precision: 0 }" />
-              </div>
-            </div>
-            <!-- /ko -->
-            <!-- ko if: typeof driver_memory !== 'undefined' -->
-            <div class="control-group" style="float: left;">
-              <label class="control-label">${_('Driver Memory')}</label>
+            <h4 data-bind="text: $root.getSnippetName(type())" style="clear:left;"></h4>
+
+            <!-- ko foreach: properties -->
+            <div class="control-group">
+              <label class="control-label" data-bind="name"></label>
               <div class="controls">
-                <jvm-memory-input params="value: driver_memory" />
+                <span data-bind="text: name"></span>
+                <!-- ko with: $root.getSessionProperties(name()) -->
+                  <!-- ko if: type == 'jvm' -->
+                    ## problem with ko actually, cf generated code
+                    ## <jvm-memory-input params="value: executorMemory"></jvm>
+                    <input class="input-small" type="text" data-bind="value: $parent.value" />
+                  <!-- /ko -->
+                  <!-- ko if: type == 'number' -->
+                  <input class="input-small" type="text" data-bind="value: $parent.value" />
+                  <!-- /ko -->
+                  <!-- ko if: type == 'string' -->
+                  <input class="input-small" type="text" data-bind="value: $parent.value" />
+                  <!-- /ko -->                  
+                  <!-- ko if: type == 'csv' -->
+                  <input class="input-small" type="text" data-bind="value: $parent.value" />
+                  <!-- /ko -->                  
+                <!-- /ko -->
+                <a href="javascript:void(0)" data-bind="click: function(data) { $parent.properties.remove(data) }">
+                  <i class="fa fa-minus"></i>
+                </a>
               </div>
             </div>
             <!-- /ko -->
+
+            <select data-bind="options: $root.availableSessionProperties,
+                       optionsText: 'nice_name',
+                       optionsValue: 'name',
+                       value: $root.selectedSessionProperties,
+                       optionsCaption: 'Choose...'"></select>
+            <i class="fa fa-plus" data-bind="click: function() { properties.push(ko.mapping.fromJS({'name': $root.selectedSessionProperties(), 'value': ''})); $root.selectedSessionProperties(''); }"></i>
+
             <a style="float: right;" class="btn pointer" title="${ _('Restart session') }" data-dismiss="modal" rel="tooltip" data-bind="click: function() { $root.selectedNotebook().restartSession($parent) }">
               <i class="fa fa-refresh"></i> ${ _('Recreate') }
             </a>
@@ -826,7 +821,6 @@ from django.utils.translation import ugettext as _
               <i class="fa fa-times"></i> ${ _('Close') }
             </a>
             <!-- /ko -->
-            <!-- /ko -->
             </br>
           </fieldset>
         </form>

+ 3 - 2
apps/spark/src/spark/views.py

@@ -28,7 +28,7 @@ from desktop.models import Document2, Document
 from spark.conf import LANGUAGES, LIVY_SERVER_SESSION_KIND
 from spark.decorators import check_document_access_permission,\
   check_document_modify_permission
-from spark.models import Notebook, get_api
+from spark.models import Notebook, get_api, SparkApi
 from spark.management.commands.spark_setup import Command
 
 
@@ -60,7 +60,8 @@ def notebook(request):
               'impala': _('-- Example: SELECT * FROM tablename, or press CTRL + space'),
               'hive': _('-- Example: SELECT * FROM tablename, or press CTRL + space'),
               'text': _('<h2>This is a text snippet</h2>Type your text here')
-          }
+          },
+          'session_properties': SparkApi.PROPERTIES
       }),
       'autocomplete_base_url': autocomplete_base_url,
       'is_yarn_mode': LIVY_SERVER_SESSION_KIND.get()