Преглед изворни кода

HUE-4187 [editor] Integrate batch submission in the UI

Romain Rigaux пре 9 година
родитељ
комит
8cce716

+ 1 - 0
desktop/libs/liboozie/src/liboozie/oozie_api.py

@@ -32,6 +32,7 @@ LOG = logging.getLogger(__name__)
 DEFAULT_USER = DEFAULT_USER.get()
 API_VERSION = 'v1' # Overridden to v2 for SLA
 
+
 _XML_CONTENT_TYPE = 'application/xml;charset=UTF-8'
 
 

+ 1 - 0
desktop/libs/notebook/src/notebook/api.py

@@ -165,6 +165,7 @@ def check_status(request):
 
   try:
     response['query_status'] = get_api(request, snippet).check_status(notebook, snippet)
+
     response['status'] = 0
   except SessionExpired:
     response['status'] = 'expired'

+ 4 - 1
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -97,6 +97,9 @@ def get_api(request, snippet):
   from notebook.connectors.spark_batch import SparkBatchApi
   from notebook.connectors.text import TextApi
 
+  if snippet.get('wasBatchExecuted'):
+    return OozieApi(user=request.user, request=request)
+
   interpreter = [interpreter for interpreter in get_interpreters(request.user) if interpreter['type'] == snippet['type']]
   if not interpreter:
     raise PopupException(_('Snippet type %(type)s is not configured in hue.ini') % snippet)
@@ -106,7 +109,7 @@ def get_api(request, snippet):
   if interface == 'hiveserver2':
     return HS2Api(user=request.user, request=request)
   elif interface == 'oozie':
-    return OozieApi(user=request.user, request=request, interpreter=snippet['type'])
+    return OozieApi(user=request.user, request=request)
   elif interface == 'livy':
     return SparkApi(request.user)
   elif interface == 'livy-batch':

+ 31 - 13
desktop/libs/notebook/src/notebook/connectors/oozie_batch.py

@@ -20,10 +20,9 @@ import logging
 from django.utils.translation import ugettext as _
 
 from desktop.lib.exceptions_renderable import PopupException
-from desktop.models import Document2, FilesystemException
+from desktop.models import Document2
 
-from liboozie.oozie_api import get_oozie
-from notebook.connectors.base import Api, QueryError
+from notebook.connectors.base import Api
 
 
 LOG = logging.getLogger(__name__)
@@ -57,7 +56,7 @@ class OozieApi(Api):
       raise PopupException(_('Oozie batch submission only accepts Hive queries at this time.'))
 
     # Create a managed workflow from the notebook doc
-    workflow_doc = WorkflowBuilder().create_workflow(document=notebook_doc, user=self.user, managed=True)
+    workflow_doc = WorkflowBuilder().create_workflow(document=notebook_doc, user=self.user, managed=True, name=_("Batch job for %s") % notebook_doc.name)
     workflow = Workflow(document=workflow_doc)
 
     # Submit workflow
@@ -69,28 +68,47 @@ class OozieApi(Api):
     }
 
   def check_status(self, notebook, snippet):
+    response = {}
     job_id = snippet['result']['handle']['id']
-    api = get_oozie(self.user)
-    status_resp = api.get_job_status(job_id)
-    return status_resp
+    oozie_job = check_job_access_permission(self.request, job_id)
+
+    response['status'] = 'running' if oozie_job.is_running() else 'available'
+
+    return response
 
   def fetch_result(self, notebook, snippet, rows, start_over):
-    pass
+    output = self.get_log(notebook, snippet)
+
+    return {
+        'data':  [[line] for line in output.split('\n')], # hdfs_link()
+        'meta': [{'name': 'Header', 'type': 'STRING_TYPE', 'comment': ''}],
+        'type': 'table',
+        'has_more': False,
+    }
 
   def cancel(self, notebook, snippet):
-    pass
+    job_id = snippet['result']['handle']['id']
+
+    job = check_job_access_permission(self, job_id)
+    oozie_job = check_job_edition_permission(job, self.user)
+
+    oozie_job.kill()
+
+    return {'status': 0}
 
   def get_log(self, notebook, snippet, startFrom=0, size=None):
     job_id = snippet['result']['handle']['id']
-    api = get_oozie(self.user)
-    status_resp = api.get_job_log(job_id)
+
+    oozie_job = check_job_access_permission(self.request, job_id)
+    status_resp = oozie_job.log
+
     return status_resp
 
   def progress(self, snippet, logs):
     job_id = snippet['result']['handle']['id']
 
-    oozie_workflow = check_job_access_permission(self.request, job_id)
-    return oozie_workflow.get_progress(),
+    oozie_job = check_job_access_permission(self.request, job_id)
+    return oozie_job.get_progress(),
 
   def close_statement(self, snippet):
     pass

+ 3 - 1
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -632,7 +632,8 @@
         statement: self.statement,
         properties: self.properties,
         result: self.result.getContext(),
-        database: self.database
+        database: self.database,
+        wasBatchExecuted: self.wasBatchExecuted()
       };
     };
 
@@ -678,6 +679,7 @@
       }
     };
 
+    self.wasBatchExecuted = ko.observable(typeof snippet.wasBatchExecuted != "undefined" && snippet.wasBatchExecuted != null ? snippet.wasBatchExecuted : false);
     self.lastExecuted = ko.observable(typeof snippet.lastExecuted != "undefined" && snippet.lastExecuted != null ? snippet.lastExecuted : 0);
 
     self.executingBlockingOperation = null; // A ExecuteStatement()

+ 27 - 5
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -307,7 +307,7 @@ ${ hueIcons.symbols() }
             <!-- ko ifnot: editorMode -->
               <i class="fa fa-file-text-o app-icon" style="vertical-align: middle"></i>
                 Notebook
-            <!-- /ko -->            
+            <!-- /ko -->
           </li>
 
           <!-- ko with: selectedNotebook -->
@@ -1821,9 +1821,32 @@ ${ hueIcons.symbols() }
     <a class="snippet-side-btn blue" data-bind="click: cancel, visible: status() == 'running'" title="${ _('Stop the currently running statement') }">
       <i class="fa fa-fw fa-stop snippet-side-single"></i>
     </a>
-    <a class="snippet-side-btn" data-bind="attr: {'title': $root.editorMode() && result.statements_count() > 1 ? '${ _ko('Execute next statement')}' : '${ _ko('Execute or CTRL + ENTER') }'}, click: execute, visible: status() != 'running' && status() != 'loading', css: {'blue': $parent.history().length == 0 || $root.editorMode(), 'disabled': statement() === '' }">
-      <i class="fa fa-fw fa-play snippet-side-single"></i>
-    </a>
+
+    <div class="inactive-action dropdown hover-actions pointer" data-bind="css: {'disabled': statement() === '' || status() === 'running' || status() === 'loading' }">
+      <a class="snippet-side-btn" style="padding-right:0;" href="javascript: void(0)" data-bind="attr: {'title': $root.editorMode() && result.statements_count() > 1 ? '${ _ko('Execute next statement')}' : '${ _ko('Execute or CTRL + ENTER') }'}, click: function() { wasBatchExecuted(false); execute(); }, visible: status() != 'running' && status() != 'loading', css: {'blue': $parent.history().length == 0 || $root.editorMode(), 'disabled': statement() === '' }">
+        <i class="fa fa-fw fa-play snippet-side-single"></i>
+      </a>
+      <!-- ko if: type() == 'hive' -->
+      <span data-bind="visible: status() != 'running' && status() != 'loading'">
+        <a class="dropdown-toggle snippet-side-btn" style="padding:0" data-toggle="dropdown" href="javascript: void(0)" data-bind="css: {'disabled': statement() === '', 'blue': currentQueryTab() == 'queryExplain' }">
+          <i class="fa fa-caret-down"></i>
+        </a>
+        <ul class="dropdown-menu less-padding">
+          <li>
+            <a href="javascript:void(0)" data-bind="click: function() { wasBatchExecuted(false); execute(); }, style: { color: statement() === '' || status() === 'running' || status() === 'loading' ? '#999' : ''}, css: {'disabled': statement() === '' || status() === 'running' || status() === 'loading' }" title="${ _('Execute interactively the current statement') }">
+              <i class="fa fa-fw fa-play snippet-side-single"></i> ${_('Execute')}
+            </a>
+          </li>
+          <li>
+            <a href="javascript:void(0)" data-bind="click: function() { wasBatchExecuted(true); execute(); }, css: {'disabled': statement() === '' }" title="${ _('Submit all the queries as a background batch job.') }">
+              <i class="fa fa-fw fa-send"></i> ${_('Batch')}
+            </a>
+          </li>
+        </ul>
+        </span>
+      <!-- /ko -->
+    </div>
+
     <!-- ko if: isSqlDialect -->
     <div class="inactive-action dropdown hover-actions pointer" data-bind="css: {'disabled': statement() === '' || status() === 'running' || status() === 'loading' }">
       <a class="snippet-side-btn" style="padding-right:0; padding-left: 2px;" href="javascript: void(0)" data-bind="click: explain, css: {'disabled': statement() === '' || status() === 'running' || status() === 'loading', 'blue': currentQueryTab() == 'queryExplain' }" title="${ _('Explain the current SQL query') }">
@@ -1832,7 +1855,6 @@ ${ hueIcons.symbols() }
       <a class="dropdown-toggle snippet-side-btn" style="padding:0" data-toggle="dropdown" href="javascript: void(0)" data-bind="css: {'disabled': statement() === '', 'blue': currentQueryTab() == 'queryExplain' }">
         <i class="fa fa-caret-down"></i>
       </a>
-
       <ul class="dropdown-menu less-padding">
         <li>
           <a href="javascript:void(0)" data-bind="click: explain, style: { color: statement() === '' || status() === 'running' || status() === 'loading' ? '#999' : ''}, css: {'disabled': statement() === '' || status() === 'running' || status() === 'loading' }" title="${ _('Explain the current SQL query') }">