Browse Source

HUE-2677 [spark] Submit a jar application from a Notebook

Romain Rigaux 10 years ago
parent
commit
be31ab817f

+ 1 - 1
apps/jobbrowser/src/jobbrowser/yarn_models.py

@@ -81,7 +81,7 @@ class Application(object):
     return self.api.kill(self.id)
 
   def filter_tasks(self, *args, **kwargs):
-    return []
+    pass
 
 
 class SparkJob(Application):

+ 0 - 1
apps/spark/src/spark/api.py

@@ -18,7 +18,6 @@
 import json
 import logging
 
-from django.http import HttpResponse
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _
 

+ 1 - 0
apps/spark/src/spark/conf.py

@@ -36,6 +36,7 @@ LANGUAGES = Config(
   default="""[
       {"name": "Scala", "type": "scala"},
       {"name": "Python", "type": "python"},
+      {"name": "Jar", "type": "jar"},
       {"name": "Impala SQL", "type": "impala"},
       {"name": "Hive SQL", "type": "hive"},
       {"name": "Text", "type": "text"}

+ 2 - 2
apps/spark/src/spark/models.py

@@ -81,7 +81,7 @@ class Notebook():
 
 
 def get_api(user, snippet):
-  if snippet['type'] in ('hive', 'impala', 'spark-sql'):
+  if snippet['type'] in ('hive', 'impala', 'spark-sql', 'jar'):
     return HS2Api(user)
   elif snippet['type'] == 'text':
     return TextApi(user)
@@ -294,7 +294,7 @@ class SparkApi():
     session = _get_snippet_session(notebook, snippet)
 
     try:
-      response = api.submit_statement(session['id'], snippet['statement'])
+      response = api.submit_statement(session['id'], snippet['statement']) ## --> post all props in json
       return {
           'id': response['id'],
           'has_result_set': True,

+ 1 - 1
apps/spark/src/spark/settings.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 DJANGO_APPS = ['spark']
-NICE_NAME = 'Spark Notebooks'
+NICE_NAME = 'Spark'
 MENU_INDEX = 11
 ICON = 'spark/art/icon_spark_48.png'
 

+ 10 - 2
apps/spark/src/spark/static/spark/js/spark.ko.js

@@ -119,7 +119,7 @@ var Snippet = function (vm, notebook, snippet) {
   self.codemirrorSize = ko.observable(typeof snippet.codemirrorSize != "undefined" && snippet.codemirrorSize != null ? snippet.codemirrorSize : 100);
   // self.statement_raw.extend({ rateLimit: 150 }); // Should prevent lag from typing but currently send the old query when using the key shortcut
   self.status = ko.observable(typeof snippet.status != "undefined" && snippet.status != null ? snippet.status : 'loading');
-  self.settings = ko.mapping.fromJS(typeof snippet.settings != "undefined" && snippet.settings != null ? snippet.settings : {});
+  self.properties = ko.mapping.fromJS(typeof snippet.properties != "undefined" && snippet.properties != null ? snippet.properties : {});
   self.variables = ko.observableArray([]);
   self.variableNames = ko.computed(function () {
     var matches = [];
@@ -562,7 +562,15 @@ var Notebook = function (vm, notebook) {
   };
 
   self.newSnippet = function () {
-    var _snippet = new Snippet(vm, self, {type: self.selectedSnippet(), result: {}});
+	var properties = {}
+
+    if (self.selectedSnippet() == 'jar') {
+      properties['app_jar'] = '';
+      properties['class'] = '';
+      properties['arguments'] = [];
+    }
+
+    var _snippet = new Snippet(vm, self, {type: self.selectedSnippet(), properties: properties, result: {}});
     self.snippets.push(_snippet);
 
     if (self.getSession(self.selectedSnippet()) == null) {

+ 28 - 1
apps/spark/src/spark/templates/editor.mako

@@ -301,6 +301,10 @@ ${ commonheader(_('Query'), app_name, user, "68px") | n,unicode }
   <img src="${ static('beeswax/art/icon_beeswax_48.png') }" class="snippet-icon">
   <!-- /ko -->
 
+  <!-- ko if: type() == 'jar' -->
+  <i class="fa fa-file-archive-o" class="snippet-icon"></i>
+  <!-- /ko -->
+
   <!-- ko if: type() == 'impala' -->
   <img src="${ static('impala/art/icon_impala_48.png') }" class="snippet-icon">
   <!-- /ko -->
@@ -356,7 +360,7 @@ ${ commonheader(_('Query'), app_name, user, "68px") | n,unicode }
           <a href="javascript:void(0)" data-bind="visible: $root.isEditing, click: function(){ remove($parent, $data); window.setTimeout(redrawFixedHeaders, 100);}"><i class="fa fa-times"></i></a>
         </div>
       </h2>
-      <!-- ko if: type() != 'text' -->
+      <!-- ko if: ['text', 'jar'].indexOf(type()) == -1  -->
       <div class="snippet-body">
         <div class="row-fluid">
           <div data-bind="css: editorKlass">
@@ -635,6 +639,29 @@ ${ commonheader(_('Query'), app_name, user, "68px") | n,unicode }
           <div data-bind="attr:{'id': 'editor_'+id()}, html: statement_raw, value: statement_raw, medium: {}" class="text-snippet"></div>
         </div>
       <!-- /ko -->
+      <!-- ko if: type() == 'jar' -->
+        <div class="snippet-body">
+          <input type="text" class="input-xlarge" data-bind="value: properties.app_jar" placeholder="${ _('Path to application jar, e.g. hdfs://localhost:8020/user/hue/oozie-examples.jar') }"/>
+          </br>
+          <input type="text" class="input-xlarge" data-bind="value: properties.class" placeholder="${ _('Class name of application, e.g. org.apache.oozie.example.SparkFileCopy') }"/>
+          </br>
+          <ul data-bind="foreach: properties.arguments" class="unstyled">
+            <li>
+              <input type="text" data-bind="value: $data" placeholder="${ _('e.g. 1000, market') }"/>
+              <a href="#" data-bind="click: function(){ $parent.arguments.remove(this); }">
+                <i class="fa fa-minus"></i>
+              </a>
+            </li>
+          </ul>
+          <a class="pointer" data-bind="click: function(){ $data.properties.arguments.push(''); }">
+            <i class="fa fa-plus"></i> ${ _('Add argument') }
+          </a>
+          </br>
+          <a title="${ _('Submit') }" data-bind="click: execute, visible: status() != 'running'" class="btn btn-primary disable-feedback pointer">
+            <i class="fa fa-play"></i>
+          </a>
+        </div>
+      <!-- /ko -->      
     </div>
   </div>
 </script>

+ 1 - 1
desktop/core/src/desktop/templates/common_header.mako

@@ -416,7 +416,7 @@ from django.utils.translation import ugettext as _
              <li><a href="/${apps['jobsub'].display_name}"><img src="${ static(apps['jobsub'].icon_path) }" class="app-icon"/> ${_('Job Designer')}</a></li>
            % endif
            % if 'spark' in apps:
-             <li><a href="/${apps['spark'].display_name}"><img src="${ static(apps['spark'].icon_path) }" class="app-icon"/> ${_('Spark')}</a></li>
+             <li><a href="/${apps['spark'].display_name}"><img src="${ static(apps['spark'].icon_path) }" class="app-icon"/> ${_('Spark (alpha)')}</a></li>
            % endif
          </ul>
        </li>