Prechádzať zdrojové kódy

HUE-8679 [jb] Show query profile in multi cluster multi cluster logic

Romain Rigaux 7 rokov pred
rodič
commit
1856351336

+ 12 - 6
apps/jobbrowser/src/jobbrowser/api2.py

@@ -51,10 +51,11 @@ def api_error_handler(func):
 def jobs(request, interface=None):
   response = {'status': -1}
 
+  cluster = json.loads(request.POST.get('cluster', '{}'))
   interface = json.loads(request.POST.get('interface'))
   filters = dict([(key, value) for _filter in json.loads(request.POST.get('filters', '[]')) for key, value in _filter.items() if value])
 
-  jobs = get_api(request.user, interface).apps(filters)
+  jobs = get_api(request.user, interface, cluster=cluster).apps(filters)
 
   response['apps'] = jobs['apps']
   response['total'] = jobs.get('total')
@@ -67,10 +68,12 @@ def jobs(request, interface=None):
 def job(request, interface=None):
   response = {'status': -1}
 
+  cluster = json.loads(request.POST.get('cluster', '{}'))
   interface = json.loads(request.POST.get('interface'))
   app_id = json.loads(request.POST.get('app_id'))
 
-  response_app = get_api(request.user, interface).app(app_id)
+  response_app = get_api(request.user, interface, cluster=cluster).app(app_id)
+
   if response_app.get('status') == -1 and response_app.get('message'):
     response.update(response_app)
   else:
@@ -84,6 +87,7 @@ def job(request, interface=None):
 def action(request, interface=None, action=None):
   response = {'status': -1, 'message': ''}
 
+  cluster = json.loads(request.POST.get('cluster', '{}'))
   interface = json.loads(request.POST.get('interface'))
   app_ids = json.loads(request.POST.get('app_ids'))
   operation = json.loads(request.POST.get('operation'))
@@ -92,7 +96,7 @@ def action(request, interface=None, action=None):
     return serve_403_error(request)
 
   response['operation'] = operation
-  response.update(get_api(request.user, interface).action(app_ids, operation))
+  response.update(get_api(request.user, interface, cluster=cluster).action(app_ids, operation))
 
   return JsonResponse(response)
 
@@ -101,12 +105,13 @@ def action(request, interface=None, action=None):
 def logs(request):
   response = {'status': -1}
 
+  cluster = json.loads(request.POST.get('cluster', '{}'))
   interface = json.loads(request.POST.get('interface'))
   app_id = json.loads(request.POST.get('app_id'))
   app_type = json.loads(request.POST.get('type'))
   log_name = json.loads(request.POST.get('name'))
 
-  response['logs'] = get_api(request.user, interface).logs(app_id, app_type, log_name, json.loads(request.GET.get('is_embeddable', 'false').lower()))
+  response['logs'] = get_api(request.user, interface, cluster=cluster).logs(app_id, app_type, log_name, json.loads(request.GET.get('is_embeddable', 'false').lower()))
   response['status'] = 0
 
   return JsonResponse(response)
@@ -116,16 +121,17 @@ def logs(request):
 def profile(request):
   response = {'status': -1}
 
+  cluster = json.loads(request.POST.get('cluster', '{}'))
   interface = json.loads(request.POST.get('interface'))
   app_id = json.loads(request.POST.get('app_id'))
   app_type = json.loads(request.POST.get('app_type'))
   app_property = json.loads(request.POST.get('app_property'))
   app_filters = dict([(key, value) for _filter in json.loads(request.POST.get('app_filters', '[]')) for key, value in _filter.items() if value])
 
-  api = get_api(request.user, interface)
+  api = get_api(request.user, interface, cluster=cluster)
   api._set_request(request) # For YARN
 
   response[app_property] = api.profile(app_id, app_type, app_property, app_filters)
   response['status'] = 0
 
-  return JsonResponse(response)
+  return JsonResponse(response)

+ 2 - 2
apps/jobbrowser/src/jobbrowser/apis/base_api.py

@@ -28,7 +28,7 @@ from desktop.lib.exceptions_renderable import PopupException
 LOG = logging.getLogger(__name__)
 
 
-def get_api(user, interface):
+def get_api(user, interface, cluster=None):
   from jobbrowser.apis.bundle_api import BundleApi
   from jobbrowser.apis.data_eng_api import DataEngClusterApi, DataEngJobApi
   from jobbrowser.apis.clusters import ClusterApi
@@ -42,7 +42,7 @@ def get_api(user, interface):
   if interface == 'jobs':
     return JobApi(user)
   elif interface == 'queries':
-    return QueryApi(user)
+    return QueryApi(user, cluster=cluster)
   elif interface == 'workflows':
     return WorkflowApi(user)
   elif interface == 'schedules':

+ 11 - 5
apps/jobbrowser/src/jobbrowser/apis/query_api.py

@@ -25,6 +25,7 @@ from django.utils.translation import ugettext as _
 
 from jobbrowser.apis.base_api import Api
 
+
 LOG = logging.getLogger(__name__)
 
 
@@ -34,18 +35,23 @@ try:
 except Exception, e:
   LOG.exception('Some application are not enabled: %s' % e)
 
-def _get_api(user):
-  session = Session.objects.get_session(user, application='impala')
-  server_url = _get_impala_server_url(session)
+
+def _get_api(user, cluster=None):
+  if cluster:
+    server_url = 'http://impala-coordinator%s:25000' % cluster
+  else:
+    session = Session.objects.get_session(user, application='impala')
+    server_url = _get_impala_server_url(session)
   return get_impalad_api(user=user, url=server_url)
 
+
 class QueryApi(Api):
 
-  def __init__(self, user, impala_api=None):
+  def __init__(self, user, impala_api=None, cluster=None):
     if impala_api:
       self.api = impala_api
     else:
-      self.api = _get_api(user)
+      self.api = _get_api(user, cluster)
 
   def apps(self, filters):
     kwargs = {}

+ 25 - 2
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -85,6 +85,19 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
 % if is_mini:
   <div class="mini-jb-context-selector">
+    <!-- ko component: {
+      name: 'hue-context-selector',
+      params: {
+        sourceType: 'impala',
+        compute: compute,
+        ##namespace: namespace,
+        ##availableDatabases: availableDatabases,
+        ##database: database,
+        hideDatabases: true
+      }
+    } --><!-- /ko -->
+
+    % if not conf.IS_K8S_ONLY.get():
     <!-- ko component: {
       name: 'hue-context-selector',
       params: {
@@ -94,6 +107,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
         hideLabels: true
       }
     } --><!-- /ko -->
+    % endif
   </div>
   <ul class="nav nav-pills">
     <!-- ko foreach: availableInterfaces -->
@@ -2488,6 +2502,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
         }
 
         return $.post("/jobbrowser/api/job/" + vm.interface(), {
+          cluster: ko.mapping.toJSON(vm.compute),
           app_id: ko.mapping.toJSON(self.id),
           interface: ko.mapping.toJSON(vm.interface)
         }, function (data) {
@@ -2643,6 +2658,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       self.fetchLogs = function (name) {
         name = name || 'default';
         $.post("/jobbrowser/api/job/logs?is_embeddable=${ str(is_embeddable).lower() }", {
+          cluster: ko.mapping.toJSON(vm.compute),
           app_id: ko.mapping.toJSON(self.id),
           interface: ko.mapping.toJSON(vm.interface),
           type: ko.mapping.toJSON(self.type),
@@ -2663,6 +2679,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
       self.fetchProfile = function (name, callback) {
         $.post("/jobbrowser/api/job/profile", {
+          cluster: ko.mapping.toJSON(vm.compute),
           app_id: ko.mapping.toJSON(self.id),
           interface: ko.mapping.toJSON(vm.interface),
           app_type: ko.mapping.toJSON(self.type),
@@ -2682,6 +2699,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
       self.fetchStatus = function () {
         $.post("/jobbrowser/api/job", {
+          cluster: ko.mapping.toJSON(vm.compute),
           app_id: ko.mapping.toJSON(self.id),
           interface: ko.mapping.toJSON(self.mainType)
         }, function (data) {
@@ -2982,6 +3000,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 
       self._fetchJobs = function (callback) {
         return $.post("/jobbrowser/api/jobs/" + vm.interface(), {
+          cluster: ko.mapping.toJSON(vm.compute),
           interface: ko.mapping.toJSON(vm.interface),
           filters: ko.mapping.toJSON(self.filters),
         }, function (data) {
@@ -3184,6 +3203,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       self.isMini = ko.observable(false);
 
       self.cluster = ko.observable();
+      self.compute = ko.observable();
 
       self.availableInterfaces = ko.pureComputed(function () {
         var jobsInterfaceCondition = function () {
@@ -3496,9 +3516,12 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       }, 'jobbrowser');
 
       % if is_mini:
-        huePubSub.subscribe('mini.jb.navigate', function (interface) {
+        huePubSub.subscribe('mini.jb.navigate', function (options) {
+          if (options.compute) {
+            jobBrowserViewModel.compute(options.compute);
+          }
           $('#jobsPanel .nav-pills li').removeClass('active');
-          interface = jobBrowserViewModel.isValidInterface(interface);
+          interface = jobBrowserViewModel.isValidInterface(options.section);
           $('#jobsPanel .nav-pills li[data-interface="' + interface + '"]').addClass('active');
           jobBrowserViewModel.selectInterface(interface);
         });

+ 2 - 2
desktop/core/src/desktop/api2.py

@@ -116,7 +116,7 @@ def get_context_namespaces(request, interface):
         sdx_namespaces = SdxApi(request.user).list_namespaces()
 
       # Adding "fake" namespace for cluster without one
-      sdx_namespaces.extend([_cluster for _cluster in adb_clusters if not _cluster.get('namespaceCrn') or (IS_K8S_ONLY.get() and _cluster['status'] != 'TERMINATING')])
+      sdx_namespaces.extend([_cluster for _cluster in adb_clusters if not _cluster.get('namespaceCrn') or (IS_K8S_ONLY.get() and 'TERMINAT' not in _cluster['status'])])
 
       namespaces.extend([{
           'id': namespace.get('crn', 'None'),
@@ -164,7 +164,7 @@ def get_context_computes(request, interface):
           'namespace': cluster.get('namespaceCrn', cluster.get('crn')),
           'compute_end_point': IS_K8S_ONLY.get() and '%(publicHost)s' % cluster['coordinatorEndpoint'] or '',
           'type': 'altus-dw'
-        } for cluster in dw_clusters if (cluster.get('status') == 'CREATED' and cluster.get('cdhVersion') >= 'CDH515') or (IS_K8S_ONLY.get() and cluster['status'] != 'TERMINATING')]
+        } for cluster in dw_clusters if (cluster.get('status') == 'CREATED' and cluster.get('cdhVersion') >= 'CDH515') or (IS_K8S_ONLY.get() and 'TERMINAT' not in cluster['status'])]
       )
 
     if interface == 'oozie' or interface == 'spark2':

+ 12 - 0
desktop/core/src/desktop/conf.py

@@ -1638,6 +1638,18 @@ def get_clusters(user):
       )
     )
 
+  if IS_K8S_ONLY.get():
+    clusters.append(
+      ('impala', {
+          'id': 'impala',
+          'name': 'impala',
+          'type': 'impala',
+          'interface': 'impala',
+          'server_host': 'impala'
+        }
+      )
+    )
+
   return OrderedDict(clusters)
 
 

+ 1 - 5
desktop/core/src/desktop/models.py

@@ -22,11 +22,7 @@ import os
 import urllib
 import uuid
 
-try:
-  from collections import OrderedDict
-except ImportError:
-  from ordereddict import OrderedDict # Python 2.6
-
+from collections import OrderedDict
 from itertools import chain
 
 from django.contrib.auth import models as auth_models

+ 3 - 3
desktop/core/src/desktop/templates/ko_components/ko_context_popover.mako

@@ -175,9 +175,9 @@ from metadata.conf import has_navigator
     <div class="context-popover-flex-fill" style="overflow: auto;" data-bind="with: data">
       <div style="padding: 8px">
         <div style="width:100%; text-align: center; margin-top: 30px; font-size: 100px; color: #787878;" data-bind="template: { name: 'app-icon-template', data: { icon: interpreter.type } }"></div>
-         <div style="width: 100%; margin-top: 20px; text-align:center">
-           <a style="font-size: 20px;" href="javascript:void(0)" data-bind="text: interpreter.displayName, hueLink: interpreter.page, click: function () { $parents[1].close(); }, attr: { 'title': interpreter.tooltip }"></a>
-         </div>
+          <div style="width: 100%; margin-top: 20px; text-align:center">
+            <a style="font-size: 20px;" href="javascript:void(0)" data-bind="text: interpreter.displayName, hueLink: interpreter.page, click: function () { $parents[1].close(); }, attr: { 'title': interpreter.tooltip }"></a>
+          </div>
       </div>
     </div>
   </script>

+ 2 - 2
desktop/core/src/desktop/templates/ko_components/ko_context_selector.mako

@@ -15,10 +15,10 @@
 ## limitations under the License.
 
 <%!
+from django.utils.translation import ugettext as _
+
 from desktop import conf
 from desktop.lib.i18n import smart_unicode
-
-from django.utils.translation import ugettext as _
 from desktop.views import _ko
 %>
 

+ 3 - 3
desktop/core/src/desktop/templates/ko_components/ko_dw_sidebar.mako

@@ -31,13 +31,13 @@
         <!-- /ko -->
         <!-- ko foreach: children -->
         <!-- ko if: $component.collapsed -->
-        <a role="button" class="sidebar-item" data-bind="hueLink: url, attr: { title: displayName }, css: { 'active': url === $component.activeUrl() }, tooltip: { placement: 'right' }">
+        <a role="button" class="sidebar-item" data-bind="hueLink: url, attr: { title: displayName }, css: { 'active': url === $component.activeUrl() }, tooltip: { placement: 'right' }, click: function() { if (url.startsWith('/jobbrowser')) { huePubSub.publish('context.selector.set.cluster', 'AltusV2'); } }">
           <span class="sidebar-icon with-tooltip"><!-- ko template: { name: 'app-icon-template' } --><!--/ko--></span>
           <span class="sidebar-item-name" data-bind="text: displayName"></span>
         </a>
         <!-- /ko -->
         <!-- ko ifnot: $component.collapsed -->
-        <a role="button" class="sidebar-item" data-bind="hueLink: url, attr: { title: displayName }, css: { 'active': url === $component.activeUrl() }">
+        <a role="button" class="sidebar-item" data-bind="hueLink: url, attr: { title: displayName }, css: { 'active': url === $component.activeUrl() }, click: function() { if (url.startsWith('/jobbrowser')) { huePubSub.publish('context.selector.set.cluster', 'AltusV2'); } }">
           <span class="sidebar-icon without-tooltip"><!-- ko template: { name: 'app-icon-template' } --><!--/ko--></span>
           <span class="sidebar-item-name" data-bind="text: displayName"></span>
         </a>
@@ -68,7 +68,7 @@
               { displayName: 'Editor', url: '/editor/?type=impala', icon: 'editor' },
               { displayName: 'Catalog', url: '/metastore/tables', icon: 'tables' },
               { displayName: 'Importer', url: '/indexer/importer', icon: 'hdfs' },
-              { displayName: 'Warehouses', url: '/hue/jobbrowser', icon: 'warehouses' }
+              { displayName: 'Warehouses', url: '/jobbrowser#!dataware2-clusters', icon: 'warehouses' }
             ]
           } else { // DE mode
             appCategory.children = [

+ 18 - 8
desktop/core/src/desktop/templates/ko_components/ko_job_browser_links.mako

@@ -19,7 +19,7 @@ from django.utils.translation import ugettext as _
 
 from notebook.conf import ENABLE_QUERY_SCHEDULING
 
-from desktop.conf import IS_EMBEDDED, IS_MULTICLUSTER_ONLY
+from desktop.conf import IS_EMBEDDED, IS_MULTICLUSTER_ONLY, IS_K8S_ONLY
 from desktop.lib.i18n import smart_unicode
 from desktop.views import _ko
 %>
@@ -28,9 +28,15 @@ from desktop.views import _ko
 
   <script type="text/html" id="hue-job-browser-links-template">
     <div class="btn-group pull-right">
-      <a class="btn btn-flat" style="padding-right: 4px" title="${_('Job browser')}" data-bind="hueLink: '/jobbrowser#!jobs', click: function() { huePubSub.publish('hide.jobs.panel'); }">
-        <span>${ _('Queries') if IS_EMBEDDED.get() or IS_MULTICLUSTER_ONLY.get() else _('Jobs') }</span>
-      </a>
+      % if IS_K8S_ONLY.get():
+        <a class="btn btn-flat" style="padding-right: 4px" title="${_('Query browser')}" data-bind="hueLink: '/jobbrowser#!queries', click: function() { huePubSub.publish('show.jobs.panel', {interface: 'queries'}); huePubSub.publish('hide.jobs.panel'); }">
+          <span>${ _('Queries') }</span>
+        </a>
+      % else:
+        <a class="btn btn-flat" style="padding-right: 4px" title="${_('Job browser')}" data-bind="hueLink: '/jobbrowser#!jobs', click: function() { huePubSub.publish('hide.jobs.panel'); }">
+          <span>${ _('Queries') if IS_EMBEDDED.get() or IS_MULTICLUSTER_ONLY.get() else _('Jobs') }</span>
+        </a>
+      % endif
       <button class="btn btn-flat btn-toggle-jobs-panel" title="${_('Jobs preview')}" data-bind="click: function() { huePubSub.publish('toggle.jobs.panel'); }, style: {'paddingLeft': jobCount() > 0 ? '0': '4px'}">
         <span class="jobs-badge" data-bind="visible: jobCount() > 0, text: jobCount"></span>
         <i class="fa fa-tasks"></i>
@@ -59,14 +65,18 @@ from desktop.views import _ko
           $jobsPanel.hide();
         });
 
-        huePubSub.subscribe('show.jobs.panel', function (section) {
+        huePubSub.subscribe('show.jobs.panel', function (options) {
+          if ('${ IS_K8S_ONLY.get() }' == 'True') {
+            huePubSub.publish('context.selector.set.cluster', 'impala');
+          };
+          
           huePubSub.publish('hide.history.panel');
           $(window).on('resize', reposition);
           reposition();
           $jobsPanel.show();
-          huePubSub.publish('mini.jb.navigate', section && section.interface ? section.interface : 'jobs');
-          if (section && section.id) {
-            huePubSub.publish('mini.jb.open.job', section.id);
+          huePubSub.publish('mini.jb.navigate', {section: options && options.interface ? options.interface : 'jobs', compute: options && options.compute});
+          if (options && options.id) {
+            huePubSub.publish('mini.jb.open.job', options.id);
           }
         });
 

+ 1 - 1
desktop/libs/notebook/src/notebook/templates/editor_components.mako

@@ -667,7 +667,7 @@ ${ sqlSyntaxDropdown.sqlSyntaxDropdown() }
       <ul data-bind="visible: jobs().length > 0, foreach: jobs" class="unstyled jobs-overlay">
         <li data-bind="attr: {'id': $data.name.substr(4)}">
           % if is_embeddable:
-            <a class="pointer" data-bind="text: $.trim($data.name), click: function() { if ($parent.compute()) { huePubSub.publish('context.selector.set.cluster', $parent.compute().name) }; huePubSub.publish('show.jobs.panel', {id: $data.name, interface: $parent.type() == 'impala' ? 'queries' : 'jobs'}); }, clickBubble: false"></a>
+            <a class="pointer" data-bind="text: $.trim($data.name), click: function() { huePubSub.publish('show.jobs.panel', {id: $data.name, interface: $parent.type() == 'impala' ? 'queries' : 'jobs', compute: $parent.compute() ? $parent.compute().name : null}); }, clickBubble: false"></a>
           % else:
             <a data-bind="text: $.trim($data.name), hueLink: $data.url"></a>
           % endif