Bläddra i källkod

HUE-8758 [connectors] Port check config to support hiveserver2 interface

Romain 5 år sedan
förälder
incheckning
7d70f9d132

+ 8 - 4
apps/about/src/about/templates/admin_wizard.mako

@@ -41,7 +41,11 @@ ${ layout.menubar(section='quick_start') }
         % if is_admin(user):
           ${ _('Quick Start Wizard') } -
         % endif
-        Hue&trade; ${version} - <a href="http://gethue.com" target="_blank" style="color:#777" title="${ _('Open gethue.com in a new window.') }">${ _("Query. Explore. Repeat.") }</a>
+        Hue&trade; ${version} -
+        Query. Explore. Repeat.
+        <a href="gethue.com" target="_blank" style="color:#777" title="${ _('Open in a new tab') }">
+          gethue.com
+        </a>
       </h1>
 
      % if is_admin(user):
@@ -50,7 +54,7 @@ ${ layout.menubar(section='quick_start') }
 
            <div class="span2">
             <ul class="nav nav-pills nav-vertical-pills">
-              <li class="active"><a href="#step1" class="step">${ _('Step 1:') } <i class="fa fa-cogs"></i> ${ _('Check Configuration') }</a></li>
+              <li class="active"><a href="#step1" class="step">${ _('Step 1:') } <i class="fa fa-cogs"></i> ${ _('Checks') }</a></li>
               <li><a href="#step2" class="step">${ _('Step 2:') } <i class="fa fa-exchange"></i> ${ _('Connectors') }</a></li>
               <li><a href="#step3" class="step">${ _('Step 3:') } <i class="fa fa-book"></i> ${ _('Examples') }</a></li>
               <li><a id="lastStep" href="#step4" class="step">${ _('Step 4:') } <i class="fa fa-group"></i> ${ _('Users') }</a></li>
@@ -79,7 +83,7 @@ ${ layout.menubar(section='quick_start') }
             % else:
               <a href="${ url('desktop.views.dump_config') }" target="_blank">${ _('Configuration') }</a>
               <br>
-              <a href="https://docs.gethue.com/latest/administrator/configuration/" target="_blank">${ _('Documentation') }</a>
+              <a href="https://docs.gethue.com/administrator/configuration/" target="_blank">${ _('Documentation') }</a>
             % endif
           </div>
 
@@ -200,7 +204,7 @@ ${ layout.menubar(section='quick_start') }
                 <input id="updateSkipWizard" type="checkbox"
                        style="margin-right: 10px"
                        title="${ _('Check to skip this wizard next time.') }"/>
-                ${ _('Skip the Quick Start Wizard at next login and land directly on the home page.') }
+                ${ _('Skip the Quick Start Wizard at next login and land directly on your starred application.') }
               </label>
             </div>
             % endif

+ 3 - 1
apps/beeswax/src/beeswax/server/dbms.py

@@ -480,7 +480,9 @@ class HiveServer2Dbms(object):
 
     # Filter on max # of partitions for partitioned tables
     column = '`%s`' % column if column else '*'
-    if table.partition_keys:
+    if operation == 'hello':
+      hql = "SELECT 'Hello World!'"
+    elif table.partition_keys:
       hql = self._get_sample_partition_query(database, table, column, limit, operation)
     elif self.server_name.startswith('impala'):
       if column or nested:

+ 16 - 14
apps/hive/src/hive/conf.py

@@ -22,6 +22,7 @@ import beeswax.hive_site
 
 from django.utils.translation import ugettext_lazy as _t, ugettext as _
 
+from desktop.conf import has_connectors
 from desktop.lib.exceptions import StructuredThriftTransportException
 from beeswax.settings import NICE_NAME
 
@@ -29,22 +30,24 @@ from beeswax.settings import NICE_NAME
 LOG = logging.getLogger(__name__)
 
 
-'''
-v2
-When using the connectors, now 'hive' is seen as a dialect and only the list of connections
-(instance of the 'hive' connector, e.g. pointing to a Hive server in the Cloud) should be tested.
-The Editor/Notebook app is the one testing it.
-
-v1
-All the configuration happens in apps/beeswax.
-'''
-
 def config_validator(user):
-  # dbms is dependent on beeswax.conf, import in method to avoid circular dependency
-  from beeswax.design import hql_query
+  '''
+  v2
+  When using the connectors, now 'hive' is seen as a dialect and only the list of connections
+  (instance of the 'hive' connector, e.g. pointing to a Hive server in the Cloud) should be tested.
+  Interpreters are now tested by the Editor in libs/notebook/conf.py.
+
+  v1
+  All the configuration happens in apps/beeswax.
+  '''
+  from beeswax.design import hql_query # dbms is dependent on beeswax.conf, import in method to avoid circular dependency
   from beeswax.server import dbms
 
   res = []
+
+  if has_connectors():
+    return res
+
   try:
     try:
       if not 'test' in sys.argv:  # Avoid tests hanging
@@ -76,7 +79,6 @@ def config_validator(user):
   except Exception:
     msg = 'Failed to access Hive warehouse: %s'
     LOG.exception(msg % warehouse)
-
-    return [(NICE_NAME, _(msg) % warehouse)]
+    res.append((NICE_NAME, _(msg) % warehouse))
 
   return res

+ 9 - 5
apps/impala/src/impala/conf.py

@@ -17,12 +17,13 @@
 
 import logging
 import os
-import sys
 import socket
+import sys
 
 from django.utils.translation import ugettext_lazy as _t, ugettext as _
-from desktop.conf import default_ssl_cacerts, default_ssl_validate, AUTH_USERNAME as DEFAULT_AUTH_USERNAME,\
-  AUTH_PASSWORD as DEFAULT_AUTH_PASSWORD
+
+from desktop.conf import default_ssl_cacerts, default_ssl_validate, AUTH_USERNAME as DEFAULT_AUTH_USERNAME, \
+    AUTH_PASSWORD as DEFAULT_AUTH_PASSWORD, has_connectors
 from desktop.lib.conf import ConfigSection, Config, coerce_bool, coerce_csv, coerce_password_from_script
 from desktop.lib.exceptions import StructuredThriftTransportException
 from desktop.lib.paths import get_desktop_root
@@ -236,13 +237,16 @@ USE_SASL = Config(
 
 
 def config_validator(user):
-  # dbms is dependent on beeswax.conf (this file)
-  # import in method to avoid circular dependency
+  # dbms is dependent on beeswax.conf, import in method to avoid circular dependency
   from beeswax.design import hql_query
   from beeswax.server import dbms
   from beeswax.server.dbms import get_query_server_config
 
   res = []
+
+  if has_connectors():
+    return res
+
   try:
     try:
       if not 'test' in sys.argv: # Avoid tests hanging

+ 0 - 1
apps/impala/src/impala/dbms.py

@@ -28,7 +28,6 @@ from beeswax.models import QUERY_TYPES
 from beeswax.server import dbms
 from beeswax.server.dbms import HiveServer2Dbms, QueryServerException, QueryServerTimeoutException,\
   get_query_server_config as beeswax_query_server_config, get_query_server_config_via_connector
-from notebook.conf import get_ordered_interpreters
 
 from impala import conf
 

+ 38 - 25
desktop/core/src/desktop/templates/check_config.mako

@@ -13,33 +13,46 @@
 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
+
 <%!
 from django.utils.translation import ugettext as _
+
+from desktop.conf import has_connectors
+from desktop.auth.backend import is_admin
 %>
-    ${_('Configuration files located in')} <code style="color: #0B7FAD">${conf_dir}</code>
 
-    <br/><br/>
-    % if error_list:
-      <div class="alert alert-warn">${_('Potential misconfiguration detected. Fix and restart Hue.')}</div>
-      <br/>
-        <table class="table table-condensed">
-      % for error in error_list:
-        <tr>
-            <td width="15%">
-                <code>
-                ${error['name'] | n}
-              </code>
-            </td>
-            <td>
-              ## Doesn't make sense to print the value of a BoundContainer
-              % if 'value' in error:
-                ${_('Current value:')} <code>${error['value']}</code><br/>
-              % endif
-              ${error['message'] | n}
-            </td>
-        </tr>
-      % endfor
-    </table>
-    % else:
-      <h5>${_('All OK. Configuration check passed.')}</h5>
+% if is_admin(user):
+  ${ _('Configuration files located in') } <code style="color: #0B7FAD">${ conf_dir }</code>
+% endif
+
+<br/><br/>
+
+% if error_list:
+  <div class="alert alert-warn">
+    ${ _('Potential misconfiguration detected.') }
+    % if not has_connectors():
+      ${ _('Fix and restart Hue.') }
     % endif
+  </div>
+  <br/>
+  <table class="table table-condensed">
+  % for error in error_list:
+    <tr>
+      <td width="15%">
+        <code>
+          ${ error['name'] | n }
+        </code>
+      </td>
+      <td>
+        ## Doesn't make sense to print the value of a BoundContainer
+        % if 'value' in error:
+          ${ _('Current value:') } <code>${ error['value'] }</code><br/>
+        % endif
+        ${ error['message'] | n }
+      </td>
+    </tr>
+  % endfor
+  </table>
+% else:
+  <h5>${ _('All OK. Configuration check passed.') }</h5>
+% endif

+ 49 - 1
desktop/libs/notebook/src/notebook/conf.py

@@ -15,15 +15,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+import logging
+
 from collections import OrderedDict
 
-from django.utils.translation import ugettext_lazy as _t
+from django.test.client import Client
+from django.utils.translation import ugettext_lazy as _t, ugettext as _
 
 from desktop import appmanager
 from desktop.conf import is_oozie_enabled, has_connectors
 from desktop.lib.conf import Config, UnspecifiedConfigSection, ConfigSection, coerce_json_dict, coerce_bool, coerce_csv
 
 
+LOG = logging.getLogger(__name__)
+
+
 SHOW_NOTEBOOKS = Config(
     key="show_notebooks",
     help=_t("Show the notebook menu or not"),
@@ -293,3 +300,44 @@ def _default_interpreters(user):
     ))
 
   INTERPRETERS.set_for_testing(OrderedDict(interpreters))
+
+
+def config_validator(user):
+  from notebook.models import _excute_test_query
+
+  res = []
+
+  if not has_connectors():
+    return res
+
+  client = Client()
+  client.force_login(user=user)
+
+  if not user.is_authenticated():
+    res.append(('Editor', _('Could not authenticate with user %s to validate interpreters') % user))
+
+  for interpreter in get_ordered_interpreters(user=user):
+    if interpreter['interface'] == 'hiveserver2':  # TODO: switch to is_sql when SqlAlchmy is ported
+      connector = interpreter['type']
+
+      try:
+        response = _excute_test_query(client, connector)
+        data = json.loads(response.content)
+
+        if data['status'] != 0:
+          raise Exception(data['message'])
+      except Exception as e:
+        trace = str(e)
+        msg = "Testing the connector connection failed."
+        if 'Error validating the login' in trace or 'TSocket read 0 bytes' in trace:
+          msg += ' Failed to authenticate, check authentication configurations.'
+
+        LOG.exception(msg)
+        res.append(
+          (
+            connector,
+            _(msg) + (' %s' % trace[:100] + ('...' if len(trace) > 50 else ''))
+          )
+        )
+
+  return res

+ 71 - 0
desktop/libs/notebook/src/notebook/conf_tests.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from nose.tools import assert_equal, assert_true, assert_false
+
+from desktop.auth.backend import rewrite_user
+from desktop.lib.connectors.api import _get_installed_connectors
+from desktop.lib.django_test_util import make_logged_in_client
+from useradmin.models import User, update_app_permissions, get_default_user_group
+
+from notebook.conf import config_validator
+
+
+if sys.version_info[0] > 2:
+  from unittest.mock import patch, Mock
+else:
+  from mock import patch, Mock
+
+
+class TestCheckConfig():
+
+  def setUp(self):
+    self.client = make_logged_in_client(
+        username='test_check_config',
+        groupname=get_default_user_group(),
+        recreate=True,
+        is_superuser=False
+    )
+    self.user = User.objects.get(username='test_check_config')
+    self.user = rewrite_user(self.user)
+
+  @patch('desktop.lib.connectors.models.CONNECTOR_INSTANCES', None)
+  @patch('notebook.conf.has_connectors', return_value=True)
+  def test_config_validator(self, has_connectors):
+
+    with patch('desktop.lib.connectors.models.CONNECTORS.get') as CONNECTORS:
+      CONNECTORS.return_value = {
+        'hive-1': Mock(
+          NICE_NAME=Mock(get=Mock(return_value='Hive')),
+          DIALECT=Mock(get=Mock(return_value='hive')),
+          INTERFACE=Mock(get=Mock(return_value='hiveserver2')),
+          SETTINGS=Mock(get=Mock(return_value=[{"name": "server_host", "value": "gethue"}, {"name": "server_port", "value": "10000"}])),
+        )
+      }
+
+      update_app_permissions()
+
+      connectors = _get_installed_connectors(user=self.user)
+      assert_true(connectors, connectors)
+
+      warnings = config_validator(user=self.user)
+
+      assert_true(warnings, warnings)
+      assert_equal('hive-1', warnings[0][0])
+      assert_true('Testing the connector connection failed' in warnings[0][1], warnings)

+ 38 - 3
desktop/libs/notebook/src/notebook/models.py

@@ -17,8 +17,7 @@
 
 from future import standard_library
 standard_library.install_aliases()
-from builtins import str
-from builtins import object
+from builtins import str, object
 import datetime
 import json
 import logging
@@ -32,9 +31,11 @@ from datetime import timedelta
 from django.contrib.sessions.models import Session
 from django.db.models import Count
 from django.db.models.functions import Trunc
+from django.urls import reverse
 from django.utils.html import escape
 from django.utils.translation import ugettext as _
 
+
 from desktop.conf import has_connectors, TASK_SERVER
 from desktop.lib.i18n import smart_unicode
 from desktop.lib.paths import SAFE_CHARACTERS_URI
@@ -44,7 +45,6 @@ from useradmin.models import User
 from notebook.connectors.base import Notebook, get_api as _get_api, get_interpreter
 
 if sys.version_info[0] > 2:
-  import urllib.request, urllib.error
   from urllib.parse import quote as urllib_quote
 else:
   from urllib import quote as urllib_quote
@@ -492,6 +492,41 @@ def _get_editor_type(editor_id):
   return document.type.rsplit('-', 1)[-1]
 
 
+def _excute_test_query(client, interpreter):
+  '''
+  Helper utils until the API gets simplified.
+  '''
+  notebook_json = """
+    {
+      "selectedSnippet": "hive",
+      "showHistory": false,
+      "description": "Test Hive Query",
+      "name": "Test Hive Query",
+      "sessions": [
+          {
+              "type": "hive",
+              "properties": [],
+              "id": null
+          }
+      ],
+      "type": "hive",
+      "id": null,
+      "snippets": [{"id":"2b7d1f46-17a0-30af-efeb-33d4c29b1055","type":"%(connector)s","status":"running","statement":"select * from web_logs","properties":{"settings":[],"variables":[],"files":[],"functions":[]},"result":{"id":"b424befa-f4f5-8799-a0b4-79753f2552b1","type":"table","handle":{"log_context":null,"statements_count":1,"end":{"column":21,"row":0},"statement_id":0,"has_more_statements":false,"start":{"column":0,"row":0},"secret":"rVRWw7YPRGqPT7LZ/TeFaA==an","has_result_set":true,"statement":"select * from web_logs","operation_type":0,"modified_row_count":null,"guid":"7xm6+epkRx6dyvYvGNYePA==an"}},"lastExecuted": 1462554843817,"database":"default"}],
+      "uuid": "d9efdee1-ef25-4d43-b8f9-1a170f69a05a"
+  }
+  """ % {
+    'connector': interpreter
+  }
+
+  return client.post(
+    reverse('notebook:api_sample_data', kwargs={'database': 'default', 'table': 'default'}), {
+      'notebook': notebook_json,
+      'snippet': json.dumps(json.loads(notebook_json)['snippets'][0]),
+      'is_async': json.dumps(True),
+      'operation': json.dumps('hello')
+  })
+
+
 class ApiWrapper(object):
   def __init__(self, request, snippet):
     self.request = request

+ 3 - 1
desktop/libs/notebook/src/notebook/sql_utils.py

@@ -14,6 +14,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from future import standard_library
 standard_library.install_aliases()
 import hashlib
@@ -28,6 +29,7 @@ if sys.version_info[0] > 2:
 else:
   from StringIO import StringIO as string_io
 
+
 # Note: Might be replaceable by sqlparse.split
 def get_statements(hql_query):
   hql_query = strip_trailing_semicolon(hql_query)
@@ -164,4 +166,4 @@ def strip_trailing_semicolon(query):
   if len(s) > 1:
     assert len(s) == 2
     assert s[1] == ''
-  return s[0]
+  return s[0]