Эх сурвалжийг харах

HUE-6245 [cluster] Clean-up imports and hardcoded hive sources

Romain Rigaux 8 жил өмнө
parent
commit
7d1222e

+ 2 - 2
apps/beeswax/src/beeswax/server/dbms.py

@@ -25,7 +25,7 @@ from django.utils.translation import ugettext as _
 
 from desktop.lib.django_util import format_preserving_redirect
 from desktop.lib.parameterization import substitute_variables
-from desktop.models import Cluster
+from desktop.models import Cluster, IMPALAUI
 from filebrowser.views import location_to_url
 
 from beeswax import hive_site
@@ -49,7 +49,7 @@ def get(user, query_server=None):
 
   if query_server is None:
     cluster_type = Cluster(user).get_type()
-    if cluster_type == 'impalaui':
+    if cluster_type == IMPALAUI:
       kwargs = {'name': 'impala'}
     else:
       kwargs = {}

+ 2 - 42
apps/jobbrowser/src/jobbrowser/apis/data_eng_api.py

@@ -16,14 +16,12 @@
 # limitations under the License.
 
 import logging
-import json
 
 from datetime import datetime,  timedelta
 
 from django.utils.translation import ugettext as _
 
-from jobbrowser.apis.base_api import Api, MockDjangoRequest, _extract_query_params
-from liboozie.oozie_api import get_oozie
+from jobbrowser.apis.base_api import Api
 from notebook.connectors.dataeng import DataEng, DATE_FORMAT
 
 
@@ -33,27 +31,6 @@ LOG = logging.getLogger(__name__)
 class DataEngClusterApi(Api):
 
   def apps(self, filters):
-#     kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
-#
-#     text_filters = _extract_query_params(filters)
-#
-#     if not has_dashboard_jobs_access(self.user):
-#       kwargs['filters'].append(('user', self.user.username))
-#     elif 'user' in text_filters:
-#       kwargs['filters'].append(('user', text_filters['username']))
-#
-#     if 'time' in filters:
-#       kwargs['filters'].extend([('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1]))])
-#
-#     if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
-#       kwargs['filters'].extend([('text', text_filters.get('text'))])
-#
-#     if filters.get('states'):
-#       states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],}
-#       for _state in filters.get('states'):
-#         for _status in states_filters[_state]:
-#           kwargs['filters'].extend([('status', _status)])
-
     api = DataEng(self.user)
 
     jobs = api.list_clusters()
@@ -102,14 +79,7 @@ class DataEngJobApi(Api):
 
   def apps(self, filters):
     kwargs = {}
-#
-#     text_filters = _extract_query_params(filters)
-#
-#     if not has_dashboard_jobs_access(self.user):
-#       kwargs['filters'].append(('user', self.user.username))
-#     elif 'user' in text_filters:
-#       kwargs['filters'].append(('user', text_filters['username']))
-#
+
     if 'time' in filters:
       if filters['time']['time_unit'] == 'minutes':
         delta = timedelta(minutes=int(filters['time']['time_value']))
@@ -119,15 +89,6 @@ class DataEngJobApi(Api):
         delta = timedelta(days=int(filters['time']['time_value']))
       kwargs['creation_date_after'] = (datetime.today() - delta).strftime(DATE_FORMAT)
 
-#     if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
-#       kwargs['filters'].extend([('text', text_filters.get('text'))])
-#
-#     if filters.get('states'):
-#       states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],}
-#       for _state in filters.get('states'):
-#         for _status in states_filters[_state]:
-#           kwargs['filters'].extend([('status', _status)])
-
     api = DataEng(self.user)
 
     jobs = api.list_jobs(**kwargs)
@@ -188,4 +149,3 @@ class DataEngJobApi(Api):
       return 'SUCCEEDED'
     else:
       return 'FAILED' # INTERRUPTED , KILLED, TERMINATED and FAILED
-

+ 7 - 7
apps/metastore/src/metastore/templates/metastore.mako

@@ -102,7 +102,7 @@ ${ components.menubar(is_embeddable) }
     <!-- ko if: editingTable -->
       <!-- ko with: table -->
       <li class="editable-breadcrumb-input">
-        <input type="text" data-bind="hivechooser: {data: name, database: $parent.name, skipColumns: true, searchEverywhere: true, onChange: function(val){ $parent.setTableByName(val); $parent.editingTable(false); }, apiHelperUser: '${ user }', apiHelperType: 'hive'}" autocomplete="off" />
+        <input type="text" data-bind="hivechooser: {data: name, database: $parent.name, skipColumns: true, searchEverywhere: true, onChange: function(val){ $parent.setTableByName(val); $parent.editingTable(false); }, apiHelperUser: '${ user }', apiHelperType: sourceType()}" autocomplete="off" />
       </li>
       <!-- /ko -->
     <!-- /ko -->
@@ -206,7 +206,7 @@ ${ components.menubar(is_embeddable) }
         <td data-bind="text: $index() + 1"></td>
         <td title="${_('Query partition data')}">
           <!-- ko if: IS_HUE_4 -->
-            <a data-bind="click: function() { queryAndWatch(notebookUrl); }, text: '[\'' + columns.join('\',\'') + '\']'" href="javascript:void(0)"></a>
+            <a data-bind="click: function() { queryAndWatch(notebookUrl, sourceType()); }, text: '[\'' + columns.join('\',\'') + '\']'" href="javascript:void(0)"></a>
           <!-- /ko -->
           <!-- ko if: ! IS_HUE_4 -->
             <a data-bind="attr: {'href': readUrl }, text: '[\'' + columns.join('\',\'') + '\']'"></a>
@@ -428,7 +428,7 @@ ${ components.menubar(is_embeddable) }
             <!-- ko if: $root.navigatorEnabled()  -->
             <h4>${ _('Tags') }</h4>
             <div style="margin-top: 5px" data-bind="component: { name: 'nav-tags', params: {
-              sourceType: 'hive',
+              sourceType: sourceType(),
               database: db_name
               } }"></div>
             <!-- /ko -->
@@ -452,7 +452,7 @@ ${ components.menubar(is_embeddable) }
         <div class="actionbar-actions" data-bind="visible: tables().length > 0, dockable: { scrollable: '${ MAIN_SCROLLABLE }', nicescroll: true, jumpCorrection: 5 }">
           <input class="input-xlarge search-query margin-left-10" type="text" placeholder="${ _('Search for a table...') }" data-bind="clearable: tableQuery, value: tableQuery, valueUpdate: 'afterkeydown'"/>
           <button class="btn toolbarBtn margin-left-20" title="${_('Browse the selected table')}" data-bind="click: function () { setTable(selectedTables()[0]); selectedTables([]); }, disable: selectedTables().length !== 1"><i class="fa fa-eye"></i> ${_('View')}</button>
-          <button class="btn toolbarBtn" title="${_('Query the selected table')}" data-bind="click: function () { IS_HUE_4 ? queryAndWatch('/notebook/browse/' + name + '/' + selectedTables()[0].name + '/') : location.href = '/notebook/browse/' + name + '/' + selectedTables()[0].name; }, disable: selectedTables().length !== 1">
+          <button class="btn toolbarBtn" title="${_('Query the selected table')}" data-bind="click: function () { IS_HUE_4 ? queryAndWatch('/notebook/browse/' + name + '/' + selectedTables()[0].name + '/', sourceType()) : location.href = '/notebook/browse/' + name + '/' + selectedTables()[0].name; }, disable: selectedTables().length !== 1">
             <i class="fa fa-play fa-fw"></i> ${_('Query')}
           </button>
           % if has_write_access:
@@ -610,7 +610,7 @@ ${ components.menubar(is_embeddable) }
     <!-- ko with: table -->
     % if USE_NEW_EDITOR.get():
     <!-- ko if: IS_HUE_4 -->
-      <a class="inactive-action" data-bind="tooltip: { placement: 'bottom', delay: 750 }, click: function() { queryAndWatch('/notebook/browse/' + database.name + '/' + name + '/'); }" title="${_('Query the table')}" href="javascript:void(0)"><i class="fa fa-play fa-fw"></i></a>
+      <a class="inactive-action" data-bind="tooltip: { placement: 'bottom', delay: 750 }, click: function() { queryAndWatch('/notebook/browse/' + database.name + '/' + name + '/', sourceType()); }" title="${_('Query the table')}" href="javascript:void(0)"><i class="fa fa-play fa-fw"></i></a>
     <!-- /ko -->
     <!-- ko if: ! IS_HUE_4 -->
       <a class="inactive-action" data-bind="tooltip: { placement: 'bottom', delay: 750 }, attr: { 'href': '/notebook/browse/' + database.name + '/' + name }" title="${_('Query the table')}"><i class="fa fa-play fa-fw"></i></a>
@@ -1197,10 +1197,10 @@ ${ components.menubar(is_embeddable) }
     });
   }
 
-  function queryAndWatch(url) {
+  function queryAndWatch(url, sourceType) {
     $.post(url, {
       format: "json",
-      sourceType: '${ source_type }'
+      sourceType: sourceType
     },function(resp) {
       if (resp.history_uuid) {
         huePubSub.publish('open.editor.query', resp.history_uuid);

+ 8 - 8
desktop/core/src/desktop/models.py

@@ -21,7 +21,6 @@ import logging
 import os
 import urllib
 import uuid
-from desktop.conf import get_clusters
 
 try:
   from collections import OrderedDict
@@ -47,6 +46,7 @@ from dashboard.conf import IS_ENABLED as IS_DASHBOARD_ENABLED
 from notebook.conf import SHOW_NOTEBOOKS, get_ordered_interpreters
 
 from desktop import appmanager
+from desktop.conf import get_clusters
 from desktop.lib.i18n import force_unicode
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.paths import get_run_root
@@ -1561,7 +1561,7 @@ class ClusterConfig():
   def get_config(self):
     app_config = self.get_apps()
     editors = app_config.get('editor')
-  
+
     return {
       'app_config': app_config,
       'main_button_action': self.get_main_quick_action(app_config),
@@ -1821,27 +1821,27 @@ class Cluster():
       clusters = get_clusters()
       cluster_name = json.loads(self.default_cluster[USER_PREFERENCE_CLUSTER]).get('name')
       self.data = cluster_name and clusters.get(cluster_name) and clusters[cluster_name] or None
-  
+
   def get_type(self):
     return self.data and self.data['type'] or 'ini'
 
   def get_interface(self):
     return json.loads(self.default_cluster[USER_PREFERENCE_CLUSTER]).get('interface')
 
-  def get_list_interface_indexes(self):  
+  def get_list_interface_indexes(self):
     default_cluster_index = 0
     default_cluster_interface = ''
-  
+
     clusters = get_clusters()
     default_cluster = get_user_preferences(self.user, key=USER_PREFERENCE_CLUSTER)
-  
+
     if clusters and default_cluster:
       default_cluster_json = json.loads(default_cluster[USER_PREFERENCE_CLUSTER])
       default_cluster_name = default_cluster_json.get('name')
-  
+
       default_cluster_index = default_cluster_name in clusters.keys() and clusters.keys().index(default_cluster_name) or 0
       default_cluster_interface = default_cluster_json.get('interface', '')
-      
+
     return default_cluster_index, default_cluster_interface
 
 

+ 7 - 3
desktop/libs/notebook/src/notebook/conf.py

@@ -26,6 +26,7 @@ from desktop import appmanager
 from desktop.conf import is_hue4
 from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection,\
   coerce_json_dict, coerce_bool, coerce_csv
+from desktop.models import Cluster
 
 
 def is_oozie_enabled():
@@ -196,6 +197,12 @@ def _default_interpreters(user):
         'name': 'Solr SQL', 'interface': 'solr', 'options': {}
     }),)
 
+  cluster = Cluster(user)
+  if cluster and cluster.get_type() == 'dataeng':
+    interpreters.append(('dataeng', {
+        'name': 'DataEng', 'interface': 'dataeng', 'options': {}
+    }))
+
   if 'spark' in apps:
     interpreters.extend((
       ('spark', {
@@ -204,9 +211,6 @@ def _default_interpreters(user):
       ('pyspark', {
           'name': 'PySpark', 'interface': 'livy', 'options': {}
       }),
-      ('dataeng', {
-          'name': 'DataEng', 'interface': 'dataeng', 'options': {}
-      }),
       ('r', {
           'name': 'R', 'interface': 'livy', 'options': {}
       }),

+ 7 - 7
desktop/libs/notebook/src/notebook/connectors/dataeng.py

@@ -52,7 +52,7 @@ RUNNING_STATES = ('QUEUED', 'RUNNING')
 
 
 class DataEngApi(Api):
-  
+
   def __init__(self, user, cluster_name, interpreter=None, request=None):
     Api.__init__(self, user, interpreter=interpreter, request=request)
     self.cluster_name = cluster_name
@@ -212,20 +212,20 @@ class DataEng():
       job['jobXml'] =  job_xml
 
     return self.submit_jobs(cluster_name, [{'hiveJob': job}])
-  
+
   def submit_spark_job(self):
     return _exec(['submit-jobs'])
-  
+
   def submit_yarn_job(self):
     return _exec(['submit-jobs'])
-  
+
   def submit_jobs(self, cluster_name, jobs):
     return _exec(['submit-jobs', '--cluster-name', cluster_name, '--jobs', json.dumps(jobs)])
 
   def terminate_jobs(self, job_ids):
     return _exec(['terminate-jobs', '--job-ids', job_ids])
 
-  
+
   def list_clusters(self, names=None, page_size=None, starting_token=None):
     args = ['list-clusters']
 
@@ -240,9 +240,9 @@ class DataEng():
 
   def create_cluster(self):
     return _exec(['create-cluster'])
-  
+
   def delete_cluster(self):
     return _exec(['delete-cluster'])
-  
+
   def describe_clusters(self):
     return _exec(['describe-cluster'])