Forráskód Böngészése

HUE-8713 [notebook] Add progress updates for TEZ jobs.

jdesjean 6 éve
szülő
commit
9ad1b33c9d

+ 6 - 2
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -634,8 +634,12 @@ class HiveServerClient:
     )
     )
 
 
     # HS2 does not return properties in TOpenSessionResp
     # HS2 does not return properties in TOpenSessionResp
-    if not session.get_properties():
-      session.properties = json.dumps(self.get_configuration())
+    # TEZ returns properties, but we need the configuration to detect engine
+    properties = session.get_properties()
+    if not properties or self.query_server['server_name'] == 'beeswax':
+      configuration = self.get_configuration()
+      properties.update(configuration)
+      session.properties = json.dumps(properties)
       session.save()
       session.save()
 
 
     return session
     return session

+ 6 - 0
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -2504,6 +2504,9 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           interface: ko.mapping.toJSON(vm.interface)
           interface: ko.mapping.toJSON(vm.interface)
         }, function (data) {
         }, function (data) {
           if (data.status == 0) {
           if (data.status == 0) {
+            if (data.app) {
+              huePubSub.publish('jobbrowser.data', [data.app]);
+            }
             if (callback) {
             if (callback) {
               callback(data);
               callback(data);
             };
             };
@@ -3014,6 +3017,9 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
           filters: ko.mapping.toJSON(self.filters),
           filters: ko.mapping.toJSON(self.filters),
         }, function (data) {
         }, function (data) {
           if (data.status == 0) {
           if (data.status == 0) {
+            if (data.apps && data.apps.length) {
+              huePubSub.publish('jobbrowser.data', data.apps);
+            }
             if (callback) {
             if (callback) {
               callback(data);
               callback(data);
             };
             };

+ 4 - 0
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -20,6 +20,7 @@ import binascii
 import copy
 import copy
 import hashlib
 import hashlib
 import logging
 import logging
+import json
 import re
 import re
 import StringIO
 import StringIO
 import struct
 import struct
@@ -187,6 +188,7 @@ class HS2Api(Api):
         properties = self.get_properties(lang)
         properties = self.get_properties(lang)
 
 
     response['properties'] = properties
     response['properties'] = properties
+    response['configuration'] = json.loads(session.properties)
     response['reuse_session'] = reuse_session
     response['reuse_session'] = reuse_session
     response['session_id'] = ''
     response['session_id'] = ''
 
 
@@ -643,6 +645,8 @@ DROP TABLE IF EXISTS `%(table)s`;
       session = self._get_session(notebook, 'hive')
       session = self._get_session(notebook, 'hive')
       if not session:
       if not session:
         LOG.warn('Cannot get jobs, failed to find active HS2 session for user: %s' % self.user.username)
         LOG.warn('Cannot get jobs, failed to find active HS2 session for user: %s' % self.user.username)
+      elif session.get('configuration') and session['configuration'].get('hive.execution.engine'):
+        return session['configuration'].get('hive.execution.engine')
       else:
       else:
         properties = session['properties']
         properties = session['properties']
         settings = next((prop['value'] for prop in properties if prop['key'] == 'settings'), None)
         settings = next((prop['value'] for prop in properties if prop['key'] == 'settings'), None)

+ 2 - 1
desktop/libs/notebook/src/notebook/static/notebook/js/notebook.ko.js

@@ -2183,7 +2183,7 @@ var EditorViewModel = (function() {
             });
             });
           }
           }
           if (self.status() == 'running') { // Maybe the query finished or failed in the meantime
           if (self.status() == 'running') { // Maybe the query finished or failed in the meantime
-            self.progress(data.progress)
+            self.progress(Math.max(data.progress, self.progress()))
           };
           };
         } else {
         } else {
           self._ajaxError(data);
           self._ajaxError(data);
@@ -2448,6 +2448,7 @@ var EditorViewModel = (function() {
     self.avoidClosing = false;
     self.avoidClosing = false;
 
 
     self.canSave = vm.canSave;
     self.canSave = vm.canSave;
+    self.configuration = ko.observable(typeof notebook.configuration != "undefined" && notebook.configuration != null ? notebook.configuration : null);
 
 
 
 
     self.getSession = function (session_type) {
     self.getSession = function (session_type) {

+ 20 - 12
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -3797,21 +3797,29 @@ ${ sqlSyntaxDropdown.sqlSyntaxDropdown() }
           return;
           return;
         }
         }
         if (jobs.length > 0) {
         if (jobs.length > 0) {
+          var progress = 0;
+          var parent;
           jobs.forEach(function (job) {
           jobs.forEach(function (job) {
-            if ($("#" + job.shortId).length > 0) {
-              var _job = ko.dataFor($("#" + job.shortId)[0]);
-              if (!isNaN(parseInt(job.mapsPercentComplete))) {
-                _job.percentJob(parseInt(job.mapsPercentComplete));
-              }
+            var id = job.shortId || job.id;
+            var el = $(".jobs-overlay li:contains(" + id + ')');
+            if (!el.length) {
+              return;
+            }
+            var context = ko.contextFor(el[0]);
+            parent = context.$parent;
+            var _job = context.$data;
+            progress = parseInt(job.mapsPercentComplete);
+            if (isNaN(progress)) {
+              progress = parseInt(job.progress);
+            }
+            if (!isNaN(progress)) {
+              _job.percentJob(progress);
+            } else {
+              progress = 0;
             }
             }
           });
           });
-        } else {
-          if (viewModel.selectedNotebook()) {
-            viewModel.selectedNotebook().snippets().forEach(function (snippet) {
-              snippet.jobs().forEach(function (job) {
-                job.percentJob(100);
-              });
-            });
+          if (parent && parent.jobs().length == 1) {
+            parent.progress(Math.max(progress, parent.progress()));
           }
           }
         }
         }
       }, HUE_PUB_SUB_EDITOR_ID);
       }, HUE_PUB_SUB_EDITOR_ID);