Преглед изворни кода

HUE-8591 [cluster] Remove extra debug info

Romain Rigaux пре 7 година
родитељ
комит
73a53d3

+ 4 - 11
apps/beeswax/src/beeswax/server/dbms.py

@@ -19,12 +19,12 @@ import logging
 import re
 import threading
 import time
-import socket
 
 from django.urls import reverse
 from django.utils.encoding import force_unicode
 from django.utils.translation import ugettext as _
 
+from desktop.conf import CLUSTER_ID
 from desktop.lib.django_util import format_preserving_redirect
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.parameterization import substitute_variables
@@ -39,8 +39,6 @@ from beeswax.common import apply_natural_sort
 from beeswax.design import hql_query
 from beeswax.hive_site import hiveserver2_use_ssl
 from beeswax.models import QueryHistory, QUERY_TYPES
-from desktop.conf import CLUSTER_ID
-from notebook.connectors.altus import DataWarehouse2Api
 
 
 LOG = logging.getLogger(__name__)
@@ -83,18 +81,13 @@ def get_query_server_config(name='beeswax', server=None, cluster=None):
 
   if cluster and cluster.get('id') != CLUSTER_ID.get():
     if 'altus:dataware:k8s' in cluster['id']:
-      cluster_config = {'server_host': cluster['compute_end_point'], 'name': cluster['name']}
+      compute_end_point = cluster['compute_end_point'][0] if type(cluster['compute_end_point']) == list else cluster['compute_end_point'] # TODO getting list from left assist
+      cluster_config = {'server_host': compute_end_point, 'name': cluster['name']} # TODO get port too
     else:
-      cluster_config = Cluster(user=None).get_config(cluster) # Direct cluster
+      cluster_config = Cluster(user=None).get_config(cluster['id']) # Direct cluster
   else:
     cluster_config = None
 
-  try:
-    LOG.debug("Query cluster mapping %s: %s %s" % (cluster, repr(cluster_config), socket.gethostbyaddr(cluster_config['server_host'])))
-  except Exception, e:
-    LOG.debug('Failed to get cluster %s host: %s "%s"' % (cluster, e, cluster_config))
-
-
   if name == 'impala':
     from impala.dbms import get_query_server_config as impala_query_server_config
     query_server = impala_query_server_config(cluster_config=cluster_config)

+ 1 - 1
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -606,7 +606,7 @@ class HiveServerClient:
 
     req = TOpenSessionReq(**kwargs)
     res = self._client.OpenSession(req)
-    self.coordinator_host = '' #self._client.get_coordinator_host()
+    self.coordinator_host = self._client.get_coordinator_host()
 
     if res.status is not None and res.status.statusCode not in (TStatusCode.SUCCESS_STATUS,):
       if hasattr(res.status, 'errorMessage') and res.status.errorMessage:

+ 29 - 30
apps/impala/src/impala/conf.py

@@ -204,33 +204,32 @@ DAEMON_API_USERNAME = Config(
 def config_validator(user):
   # dbms is dependent on beeswax.conf (this file)
   # import in method to avoid circular dependency
-  return None
-#   from beeswax.design import hql_query
-#   from beeswax.server import dbms
-#   from beeswax.server.dbms import get_query_server_config
-# 
-#   res = []
-#   try:
-#     try:
-#       if not 'test' in sys.argv: # Avoid tests hanging
-#         query_server = get_query_server_config(name='impala')
-#         server = dbms.get(user, query_server)
-#         query = hql_query("SELECT 'Hello World!';")
-#         handle = server.execute_and_wait(query, timeout_sec=10.0)
-# 
-#         if handle:
-#           server.fetch(handle, rows=100)
-#           server.close(handle)
-#     except StructuredThriftTransportException, ex:
-#       if 'TSocket read 0 bytes' in str(ex):  # this message appears when authentication fails
-#         msg = "Failed to authenticate to Impalad, check authentication configurations."
-#         LOG.exception(msg)
-#         res.append((NICE_NAME, _(msg)))
-#       else:
-#         raise ex
-#   except Exception, ex:
-#     msg = "No available Impalad to send queries to."
-#     LOG.exception(msg)
-#     res.append((NICE_NAME, _(msg)))
-# 
-#   return res
+  from beeswax.design import hql_query
+  from beeswax.server import dbms
+  from beeswax.server.dbms import get_query_server_config
+
+  res = []
+  try:
+    try:
+      if not 'test' in sys.argv: # Avoid tests hanging
+        query_server = get_query_server_config(name='impala')
+        server = dbms.get(user, query_server)
+        query = hql_query("SELECT 'Hello World!';")
+        handle = server.execute_and_wait(query, timeout_sec=10.0)
+
+        if handle:
+          server.fetch(handle, rows=100)
+          server.close(handle)
+    except StructuredThriftTransportException, ex:
+      if 'TSocket read 0 bytes' in str(ex):  # this message appears when authentication fails
+        msg = "Failed to authenticate to Impalad, check authentication configurations."
+        LOG.exception(msg)
+        res.append((NICE_NAME, _(msg)))
+      else:
+        raise ex
+  except Exception, ex:
+    msg = "No available Impalad to send queries to."
+    LOG.exception(msg)
+    res.append((NICE_NAME, _(msg)))
+
+  return res

+ 1 - 1
apps/jobbrowser/src/jobbrowser/apis/data_warehouse.py

@@ -111,7 +111,7 @@ class DataWarehouseClusterApi(Api):
     return {}
 
   def _api_status(self, status):
-    if status in ['CREATING', 'CREATED', 'ONLINE', 'SCALING_UP', 'SCALING_DOWN', 'STOPPED', 'STARTING']:
+    if status in ['CREATING', 'CREATED', 'ONLINE', 'SCALING_UP', 'SCALING_DOWN', 'STOPPED', 'STARTING']: # ONLINE ... are from K8s
       return 'RUNNING'
     elif status in ['ARCHIVING', 'COMPLETED', 'TERMINATING']:
       return 'SUCCEEDED'

+ 11 - 5
apps/jobbrowser/src/jobbrowser/templates/job_browser.mako

@@ -27,7 +27,7 @@ SUFFIX = is_mini and "-mini" or ""
 %>
 
 % if not is_embeddable:
-${ commonheader("Data Warehouse", "jobbrowser", user, request) | n,unicode }
+${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
 <%namespace name="assist" file="/assist.mako" />
 % endif
 
@@ -108,7 +108,15 @@ ${ commonheader("Data Warehouse", "jobbrowser", user, request) | n,unicode }
             <li class="app-header">
               <a href="/${app_name}">
                 <img src="${ static('jobbrowser/art/icon_jobbrowser_48.png') }" class="app-icon" alt="${ _('Job browser icon') }"/>
-                ${ _('Data Warehouse') }
+                <!-- ko if: !$root.cluster() || $root.cluster()['type'].indexOf("altus") == -1 -->
+                  ${ _('Job Browser') }
+                <!-- /ko -->
+                <!-- ko if: $root.cluster() && $root.cluster()['type'].indexOf("altus-dw") !== -1 -->
+                  ${ _('Data Warehouse') }
+                <!-- /ko -->
+                <!-- ko if: $root.cluster() && $root.cluster()['type'].indexOf("altus-de") !== -1 -->
+                  ${ _('Data Engineering') }
+                <!-- /ko -->
               </a>
             </li>
             <!-- ko foreach: availableInterfaces -->
@@ -1165,9 +1173,7 @@ ${ commonheader("Data Warehouse", "jobbrowser", user, request) | n,unicode }
       <div class="sidebar-nav">
         <ul class="nav nav-list">
           <li class="nav-header">${ _('Id') }</li>
-          <li><span data-bind="text: id"></span></li>
-          <li class="nav-header">${ _('Name') }</li>
-          <li><span data-bind="text: name"></span></li>
+          <li><span class="break-word" data-bind="text: id"></span></li>
           <li class="nav-header">${ _('Type') }</li>
           <li><span data-bind="text: properties['properties']['cdhVersion']"></span></li>
           <li class="nav-header">${ _('Status') }</li>

+ 1 - 0
desktop/core/src/desktop/api2.py

@@ -106,6 +106,7 @@ def get_context_namespaces(request, interface):
             _cluster['id'] = _cluster['crn']
             _cluster['namespaceName'] = _cluster['clusterName']
             _cluster['name'] = _cluster['clusterName']
+            _cluster['compute_end_point'] = '%(publicHost)s' % _cluster['coordinatorEndpoint'] if IS_K8_ONLY.get() else '',
       else:
         adb_clusters = []
 

+ 1 - 2
desktop/core/src/desktop/models.py

@@ -49,8 +49,7 @@ from kafka.conf import has_kafka
 from notebook.conf import SHOW_NOTEBOOKS, get_ordered_interpreters
 
 from desktop import appmanager
-from desktop.conf import get_clusters, CLUSTER_ID, IS_MULTICLUSTER_ONLY,\
-  IS_EMBEDDED, IS_K8_ONLY
+from desktop.conf import get_clusters, CLUSTER_ID, IS_MULTICLUSTER_ONLY, IS_EMBEDDED, IS_K8_ONLY
 from desktop.lib.i18n import force_unicode
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.lib.paths import get_run_root

+ 1 - 1
desktop/core/src/desktop/templates/assist.mako

@@ -929,7 +929,7 @@ from desktop.views import _ko
   <script type="text/html" id="assist-namespaces-template">
     <div class="assist-flex-header">
       <div class="assist-inner-header">
-        ${_('Clusters')}
+        ${_('Namespaces')}
         <!-- ko template: 'assist-namespace-header-actions' --><!-- /ko -->
       </div>
     </div>

+ 19 - 2
desktop/core/src/desktop/templates/ko_components/ko_multi_cluster_sidebar.mako

@@ -154,10 +154,10 @@ from desktop.views import _ko
             icon: 'altus-icon altus-iam',
             items: [{
                 label: '${ _('User') }',
-                url: 'https://console.altus.cloudera.com/iam/index.html#/users'
+                url: '/hue/useradmin/users/'
               },{
                 label: '${ _('Groups') }',
-                url: 'https://console.altus.cloudera.com/iam/index.html#/groups'
+                url: '/hue/useradmin/groups/'
               }
             ]
           }
@@ -169,6 +169,23 @@ from desktop.views import _ko
             url: '/',
             icon: 'altus-icon altus-sdx'
           }
+##           },{
+##             label: '${ _('Streaming') }',
+##             url: '/',
+##             icon: 'fa fa-sitemap'
+##           },{
+##             label: '${ _('Operational DB') }',
+##             url: '/',
+##             icon: 'altus-icon altus-adb',
+##             items: [{
+##                 label: '${ _('Kudu') }',
+##                 url: '/'
+##               },{
+##                 label: '${ _('HBase') }',
+##                 url: '/'
+##               }
+##             ]
+##           }
         ]
       }, {
         label: '${ _('Compute') }',

+ 8 - 7
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -320,15 +320,16 @@ def get_api(request, snippet):
   if not cluster and snippet.get('compute'): # Via notebook.ko.js
     cluster = snippet['compute']
 
-  if cluster and 'altus:dataware:k8s' in cluster:
+  cluster_name = cluster.get('id') if cluster else None
+
+  if cluster and 'altus:dataware:k8s' in cluster_name:
     interface = 'hiveserver2'
-  elif cluster and 'crn:altus:dataware:' in cluster:
+  elif cluster and 'crn:altus:dataware:' in cluster_name:
     interface = 'altus-adb'
-  elif cluster and 'crn:altus:dataeng:' in cluster:
+  elif cluster and 'crn:altus:dataeng:' in cluster_name:
     interface = 'dataeng'
 
-  if cluster:
-    LOG.info('Selected cluster %s interface %s' % (cluster, interface))
+  LOG.info('Selected cluster %s %s interface %s' % (cluster_name, cluster, interface))
 
   if interface == 'hiveserver2':
     from notebook.connectors.hiveserver2 import HS2Api
@@ -349,10 +350,10 @@ def get_api(request, snippet):
     return RdbmsApi(request.user, interpreter=snippet['type'], query_server=snippet.get('query_server'))
   elif interface == 'altus-adb':
     from notebook.connectors.altus_adb import AltusAdbApi
-    return AltusAdbApi(user=request.user, cluster_name=cluster.get('id'), request=request)
+    return AltusAdbApi(user=request.user, cluster_name=cluster_name, request=request)
   elif interface == 'dataeng':
     from notebook.connectors.dataeng import DataEngApi
-    return DataEngApi(user=request.user, request=request, cluster_name=cluster.get('id'))
+    return DataEngApi(user=request.user, request=request, cluster_name=cluster_name)
   elif interface == 'jdbc':
     if not interpreter['options'] or interpreter['options'].get('url', '').find('teradata') < 0:
       from notebook.connectors.jdbc import JdbcApi