Browse Source

HUE-2677 [spark] Support Python files in Notebook

Romain Rigaux 10 years ago
parent
commit
003085e134

+ 2 - 1
apps/spark/src/spark/conf.py

@@ -35,8 +35,9 @@ LANGUAGES = Config(
   type=coerce_json,
   default="""[
       {"name": "Scala", "type": "scala"},
-      {"name": "Python", "type": "python"},
+      {"name": "PySpark", "type": "python"},
       {"name": "Jar", "type": "jar"},
+      {"name": "Python", "type": "py"},
       {"name": "Impala SQL", "type": "impala"},
       {"name": "Hive SQL", "type": "hive"},
       {"name": "Text", "type": "text"}

+ 5 - 5
apps/spark/src/spark/models.py

@@ -83,7 +83,7 @@ class Notebook():
 def get_api(user, snippet):
   if snippet['type'] in ('hive', 'impala', 'spark-sql'):
     return HS2Api(user)
-  elif snippet['type'] == 'jar':
+  elif snippet['type'] in ('jar', 'py'):
     return SparkBatchApi(user)
   elif snippet['type'] == 'text':
     return TextApi(user)
@@ -424,10 +424,10 @@ class SparkBatchApi():
     api = get_spark_api(self.user)
 
     properties = {
-        'file': snippet['properties']['app_jar'],
-        'className': snippet['properties']['class'],
-        'args': snippet['properties']['arguments'],
-        # pyFiles
+        'file': snippet['properties'].get('app_jar'),
+        'className': snippet['properties'].get('class'),
+        'args': [arg['value'] for arg in snippet['properties'].get('arguments')],
+        'pyFiles': snippet['properties'].get('py_file'),
         # files
         # driverMemory
         # driverCores

+ 2 - 1
apps/spark/src/spark/static/spark/js/spark.ko.js

@@ -565,8 +565,9 @@ var Notebook = function (vm, notebook) {
   self.newSnippet = function () {
 	var properties = {}
 
-    if (self.selectedSnippet() == 'jar') {
+    if (self.selectedSnippet() == 'jar' || self.selectedSnippet() == 'py') {
       properties['app_jar'] = '';
+      properties['py_file'] = '';
       properties['class'] = '';
       properties['arguments'] = [];
     }

+ 13 - 8
apps/spark/src/spark/templates/editor.mako

@@ -305,6 +305,10 @@ ${ commonheader(_('Query'), app_name, user, "68px") | n,unicode }
   <i class="fa fa-file-archive-o" class="snippet-icon"></i>
   <!-- /ko -->
 
+  <!-- ko if: type() == 'py' -->
+  <i class="fa fa-file-code-o" class="snippet-icon"></i>
+  <!-- /ko -->
+
   <!-- ko if: type() == 'impala' -->
   <img src="${ static('impala/art/icon_impala_48.png') }" class="snippet-icon">
   <!-- /ko -->
@@ -360,7 +364,7 @@ ${ commonheader(_('Query'), app_name, user, "68px") | n,unicode }
           <a href="javascript:void(0)" data-bind="visible: $root.isEditing, click: function(){ remove($parent, $data); window.setTimeout(redrawFixedHeaders, 100);}"><i class="fa fa-times"></i></a>
         </div>
       </h2>
-      <!-- ko if: ['text', 'jar'].indexOf(type()) == -1  -->
+      <!-- ko if: ['text', 'jar', 'py'].indexOf(type()) == -1  -->
       <div class="snippet-body">
         <div class="row-fluid">
           <div data-bind="css: editorKlass">
@@ -639,21 +643,22 @@ ${ commonheader(_('Query'), app_name, user, "68px") | n,unicode }
           <div data-bind="attr:{'id': 'editor_'+id()}, html: statement_raw, value: statement_raw, medium: {}" class="text-snippet"></div>
         </div>
       <!-- /ko -->
-      <!-- ko if: type() == 'jar' -->
+      <!-- ko if: type() == 'jar' || type() == 'py'-->
         <div class="snippet-body">
-          <input type="text" class="input-xlarge" data-bind="value: properties.app_jar" placeholder="${ _('Path to application jar, e.g. hdfs://localhost:8020/user/hue/oozie-examples.jar') }"/>
+          <input type="text" class="input-xlarge" data-bind="value: properties.app_jar, visible: type() =='jar'" placeholder="${ _('Path to application jar, e.g. hdfs://localhost:8020/user/hue/oozie-examples.jar') }"/>
+          <input type="text" class="input-xlarge" data-bind="value: properties.py_file, visible: type() =='py'" placeholder="${ _('Path to python file, e.g. script.py') }"/>
           </br>
-          <input type="text" class="input-xlarge" data-bind="value: properties.class" placeholder="${ _('Class name of application, e.g. org.apache.oozie.example.SparkFileCopy') }"/>
+          <input type="text" class="input-xlarge" data-bind="value: properties.class, visible: type() =='jar'" placeholder="${ _('Class name of application, e.g. org.apache.oozie.example.SparkFileCopy') }"/>
           </br>
           <ul data-bind="foreach: properties.arguments" class="unstyled">
             <li>
-              <input type="text" data-bind="value: $data" placeholder="${ _('e.g. 1000, market') }"/>
-              <a href="#" data-bind="click: function(){ $parent.arguments.remove(this); }">
+              <input type="text" data-bind="value: value" placeholder="${ _('e.g. 1000, market') }"/>
+              <a href="#" data-bind="click: function(){ $parent.properties.arguments.remove(this); }">
                 <i class="fa fa-minus"></i>
               </a>
             </li>
           </ul>
-          <a class="pointer" data-bind="click: function(){ $data.properties.arguments.push(''); }">
+          <a class="pointer" data-bind="click: function(){ $data.properties.arguments.push({'value': ''}); }">
             <i class="fa fa-plus"></i> ${ _('Add argument') }
           </a>
           </br>
@@ -661,7 +666,7 @@ ${ commonheader(_('Query'), app_name, user, "68px") | n,unicode }
             <i class="fa fa-play"></i>
           </a>
         </div>
-      <!-- /ko -->      
+      <!-- /ko -->
     </div>
   </div>
 </script>

+ 1 - 1
desktop/conf.dist/hue.ini

@@ -1037,7 +1037,7 @@
   ## server_url=http://localhost:8998/
 
   # List of available types of snippets
-  ## languages='[{"name": "Scala", "type": "scala"},{"name": "Python", "type": "python"},{"name": "Impala SQL", "type": "impala"},{"name": "Hive SQL", "type": "hive"},{"name": "Text", "type": "text"}]'
+  ## languages='[{"name": "Scala", "type": "scala"},{"name": "Python", "type": "python"},{"name": "PySpark", "type": "python"},{"name": "Python", "type": "py"},{"name": "Impala SQL", "type": "impala"},{"name": "Hive SQL", "type": "hive"},{"name": "Text", "type": "text"}]'
 
 
 ###########################################################################

+ 1 - 1
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1044,7 +1044,7 @@
   ## server_url=http://localhost:8998/
 
   # List of available types of snippets
-  ## languages='[{"name": "Scala", "type": "scala"},{"name": "Python", "type": "python"},{"name": "Impala SQL", "type": "impala"},{"name": "Hive SQL", "type": "hive"},{"name": "Text", "type": "text"}]'
+  ## languages='[{"name": "Scala", "type": "scala"},{"name": "Python", "type": "python"},{"name": "PySpark", "type": "python"},{"name": "Python", "type": "py"},{"name": "Impala SQL", "type": "impala"},{"name": "Hive SQL", "type": "hive"},{"name": "Text", "type": "text"}]'
 
 
 ###########################################################################