Browse Source

HUE-8330 [jb] Listing various clusters and configurations

Romain Rigaux 7 years ago
parent
commit
3a29a50d95

+ 1 - 2
apps/jobbrowser/src/jobbrowser/apis/data_eng_api.py

@@ -21,8 +21,7 @@ from datetime import datetime,  timedelta
 
 from django.utils.translation import ugettext as _
 
-from notebook.connectors.altus import DATE_FORMAT
-from notebook.connectors.dataeng import DataEngApi
+from notebook.connectors.altus import DataEngApi, DATE_FORMAT
 
 from jobbrowser.apis.base_api import Api
 

+ 17 - 6
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -98,6 +98,11 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
               </li>
             <!-- /ko -->
           </ul>
+          <span class="pull-right">
+            <!-- ko if: availableComputes().length > 1 -->
+              <div data-bind="component: { name: 'hue-drop-down', params: { value: compute, entries: availableComputes, labelAttribute: 'name', searchable: true, linkTitle: '${ _ko('Active clusters') }' } }"></div>
+            <!-- /ko -->
+          </span>
           % if not hiveserver2_impersonation_enabled:
             <div class="pull-right label label-warning" style="margin-top: 16px">${ _("Hive jobs are running as the 'hive' user") }</div>
           % endif
@@ -2675,31 +2680,37 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
       self.clusterType = ko.observable();
       self.isMini = ko.observable(false);
 
+      self.availableComputes = ko.observableArray();
+      self.compute = ko.observable();
+    
+      ContextCatalog.getComputes({ sourceType: 'jobs' }).done(self.availableComputes);
+
       self.availableInterfaces = ko.pureComputed(function () {
         var jobsInterfaceCondition = function () {
-          return self.appConfig() && self.appConfig()['browser'] && self.appConfig()['browser']['interpreter_names'].indexOf('yarn') != -1 && self.clusterType() != '${ ANALYTIC_DB }';
+          return self.appConfig() && self.appConfig()['browser'] && self.appConfig()['browser']['interpreter_names'].indexOf('yarn') != -1 && self.clusterType() != '${ ANALYTIC_DB }' && !(self.compute() && self.compute()['type'] == 'altus');
         }
         var dataEngInterfaceCondition = function () {
-          return self.appConfig() && self.appConfig()['browser'] && self.appConfig()['browser']['interpreter_names'].indexOf('dataeng') != -1;
+          // return self.appConfig() && self.appConfig()['browser'] && self.appConfig()['browser']['interpreter_names'].indexOf('dataeng') != -1;
+          return self.compute() && self.compute()['type'] == 'altus';
         }
         var schedulerInterfaceCondition = function () {
-          return '${ user.has_hue_permission(action="access", app="oozie") }' == 'True' && self.clusterType() != '${ ANALYTIC_DB }';
+          return '${ user.has_hue_permission(action="access", app="oozie") }' == 'True' && self.clusterType() != '${ ANALYTIC_DB }' && !(self.compute() && self.compute()['type'] == 'altus');
         }
         var livyInterfaceCondition = function () {
-          return self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('pyspark') != -1;
+          return self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('pyspark') != -1 && !(self.compute() && self.compute()['type'] == 'altus');
         }
         var queryInterfaceCondition = function () {
-          return '${ ENABLE_QUERY_BROWSER.get() }' == 'True' && self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('impala') != -1;
+          return '${ ENABLE_QUERY_BROWSER.get() }' == 'True' && self.appConfig() && self.appConfig()['editor'] && self.appConfig()['editor']['interpreter_names'].indexOf('impala') != -1 && !(self.compute() && self.compute()['type'] == 'altus');
         }
 
         var interfaces = [
           {'interface': 'jobs', 'label': '${ _ko('Jobs') }', 'condition': jobsInterfaceCondition},
           {'interface': 'queries', 'label': '${ _ko('Queries') }', 'condition': queryInterfaceCondition},
-          {'interface': 'dataeng-jobs', 'label': '${ _ko('Jobs') }', 'condition': dataEngInterfaceCondition},
           {'interface': 'workflows', 'label': '${ _ko('Workflows') }', 'condition': schedulerInterfaceCondition},
           {'interface': 'schedules', 'label': '${ _ko('Schedules') }', 'condition': schedulerInterfaceCondition},
           {'interface': 'bundles', 'label': '${ _ko('Bundles') }', 'condition': schedulerInterfaceCondition},
           {'interface': 'slas', 'label': '${ _ko('SLAs') }', 'condition': schedulerInterfaceCondition},
+          {'interface': 'dataeng-jobs', 'label': '${ _ko('Jobs') }', 'condition': dataEngInterfaceCondition},
           {'interface': 'dataeng-clusters', 'label': '${ _ko('Clusters') }', 'condition': dataEngInterfaceCondition},
           {'interface': 'livy-sessions', 'label': '${ _ko('Livy') }', 'condition': livyInterfaceCondition},
         ];

+ 0 - 2
apps/oozie/src/oozie/templates/editor2/workflow_editor.mako

@@ -415,9 +415,7 @@ ${ layout.menubar(section='workflows', is_editor=True, pullright=buttons, is_emb
   <div class="container-fluid">
     <span class="pull-right">
     <!-- ko if: availableComputes().length > 1 -->
-      ##<select data-bind="selectize: availableComputes, value: compute, optionsValue: 'id', optionsText: 'name'" class="input-medium"></select>
       <div data-bind="component: { name: 'hue-drop-down', params: { value: compute, entries: availableComputes, labelAttribute: 'name', searchable: true, linkTitle: '${ _ko('Active compute') }' } }"></div>
-      ## <div class="margin-left-10" data-bind="component: { name: 'hue-drop-down', params: { icon: 'fa-snowflake-o', value: namespace, entries: availableNamespaces, labelAttribute: 'name', foreachVisible: true, searchable: true, linkTitle: '${ _ko('Namespaces') }' } }" style="display: inline-block"></div>
     <!-- /ko -->
     </span>
       

+ 2 - 2
apps/oozie/src/oozie/views/editor2.py

@@ -404,8 +404,8 @@ def _submit_workflow_helper(request, workflow, submit_action):
       if cluster.get('type') == 'altus-de':
         notebook = {}
         snippet = {'statement': 'SELECT 1'}
-        # TODO: open in Job Browser, Jobs, compute context
-        print DataEngApi(user=request.user, request=request, cluster_name=cluster.get('name')).execute(notebook, snippet)
+        handle = DataEngApi(user=request.user, request=request, cluster_name=cluster.get('name')).execute(notebook, snippet)
+        return JsonResponse({'status': 0, 'job_id': handle.get('id'), 'type': 'workflow'}, safe=False)
 
       try:
         job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping)

+ 5 - 6
desktop/core/src/desktop/api2.py

@@ -88,7 +88,7 @@ def get_context_namespaces(request, interface):
     namespaces.extend([{
         'id': cluster['id'],
         'name': cluster['name']
-      } for cluster in clusters if cluster.get('type') == 'direct' # and interfaces == 'hive'
+      } for cluster in clusters if cluster.get('type') == 'direct' # and interface == 'hive'
     ])
 
     # From Altus SDX
@@ -97,7 +97,6 @@ def get_context_namespaces(request, interface):
           'id': namespace.get('crn', 'None'),
           'name': namespace.get('namespaceName', 'Unknown'),
           'status': namespace.get('status'),
-          # creationDate
         } for namespace in SdxApi(request.user).list_namespaces()]
       )
 
@@ -114,17 +113,17 @@ def get_context_computes(request, interface):
 
   clusters = get_clusters(request.user).values()
 
-  if interface == 'hive' or interface == 'oozie':
+  if interface == 'hive' or interface == 'oozie' or interface == 'jobs':
     computes.extend([{
         'id': cluster['id'],
         'name': cluster['name'],
         'namespace': cluster['id'], # Dummy
         'interface': interface,
-        'type': 'default'
+        'type': cluster['type']
       } for cluster in clusters
     ])
 
-  if interface == 'hive':
+  if interface == 'hive' or interface == 'jobs':
     if [cluster for cluster in clusters if cluster['type'] == 'altus']:
       computes.extend([{
           'id': cluster.get('crn', 'None'),
@@ -138,7 +137,7 @@ def get_context_computes(request, interface):
         } for cluster in AnalyticDbApi(request.user).list_clusters()['clusters']]
       )
 
-  if interface == 'oozie':
+  if interface == 'oozie' or interface == 'jobs':
     if [cluster for cluster in clusters if cluster['type'] == 'altus']:
       computes.extend([{
           'id': cluster.get('crn'),