소스 검색

HUE-4969 [core] fixing Support hive.server2.thrift.sasl.qop="auth-conf"

Prakash Ranade 9 년 전
부모
커밋
5de3f14

+ 5 - 0
apps/beeswax/src/beeswax/hive_site.py

@@ -55,6 +55,8 @@ _CNF_HIVESERVER2_TRANSPORT_MODE = 'hive.server2.transport.mode'
 _CNF_HIVESERVER2_THRIFT_HTTP_PORT = 'hive.server2.thrift.http.port'
 _CNF_HIVESERVER2_THRIFT_HTTP_PATH = 'hive.server2.thrift.http.path'
 
+_CNF_HIVESERVER2_THRIFT_SASL_QOP = 'hive.server2.thrift.sasl.qop'
+
 
 # Host is whatever up to the colon. Allow and ignore a trailing slash.
 _THRIFT_URI_RE = re.compile("^thrift://([^:]+):(\d+)[/]?$")
@@ -129,6 +131,9 @@ def get_metastore_warehouse_dir():
 def get_hiveserver2_authentication():
   return get_conf().get(_CNF_HIVESERVER2_AUTHENTICATION, 'NONE').upper() # NONE == PLAIN SASL
 
+def get_hiveserver2_thrift_sasl_qop():
+  return get_conf().get(_CNF_HIVESERVER2_THRIFT_SASL_QOP, 'NONE').lower()
+
 def hiveserver2_impersonation_enabled():
   return get_conf().get(_CNF_HIVESERVER2_IMPERSONATION, 'TRUE').upper() == 'TRUE'
 

+ 94 - 1
apps/beeswax/src/beeswax/tests.py

@@ -21,9 +21,11 @@ import gzip
 import json
 import logging
 import os
+import random
 import re
 import shutil
 import socket
+import string
 import tempfile
 import threading
 
@@ -107,6 +109,8 @@ def _make_query(client, query, submission_type="Execute",
 
   return res
 
+def random_generator(size=8, chars=string.ascii_uppercase + string.digits):
+   return ''.join(random.choice(chars) for _ in range(size))
 
 def get_csv(client, result_response):
   """Get the csv for a query result"""
@@ -2966,7 +2970,6 @@ class TestDesign():
 def search_log_line(expected_log, all_logs):
   return re.compile('%(expected_log)s' % {'expected_log': expected_log}).search(all_logs)
 
-
 def test_hiveserver2_get_security():
   make_logged_in_client()
   user = User.objects.get(username='test')
@@ -3438,5 +3441,95 @@ def test_hiveserver2_jdbc_url():
     for reset in resets:
         reset()
 
+def test_sasl_auth_in_large_download():
+  db = None
+  failed = False
+  max_rows = 10000
+
+  if hive_site.get_hiveserver2_thrift_sasl_qop() != "auth-conf" or \
+     hive_site.get_hiveserver2_authentication() != 'KERBEROS':
+    raise SkipTest
+
+  client = make_logged_in_client(username="systest", groupname="systest", recreate=False, is_superuser=False)
+  user = User.objects.get(username='systest')
+  add_to_group('systest')
+  grant_access("systest", "systest", "beeswax")
+
+  desktop_conf.SASL_MAX_BUFFER.set_for_testing(2*1024*1024)
+
+  # Create a big table
+  table_info = {'db': 'default', 'table_name': 'dummy_'+random_generator().lower()}
+  drop_sql = "DROP TABLE IF EXISTS %(db)s.%(table_name)s" % table_info
+  create_sql = "CREATE TABLE IF NOT EXISTS %(db)s.%(table_name)s (w0 CHAR(8),w1 CHAR(8),w2 CHAR(8),w3 CHAR(8),w4 CHAR(8),w5 CHAR(8),w6 CHAR(8),w7 CHAR(8),w8 CHAR(8),w9 CHAR(8))" % table_info
+  hql = cStringIO.StringIO()
+  hql.write("INSERT INTO %(db)s.%(table_name)s VALUES " % (table_info))
+  for i in xrange(max_rows-1):
+    w = random_generator(size=7)
+    hql.write("('%s0','%s1','%s2','%s3','%s4','%s5','%s6','%s7','%s8','%s9')," % (w,w,w,w,w,w,w,w,w,w))
+  w = random_generator(size=7)
+  hql.write("('%s0','%s1','%s2','%s3','%s4','%s5','%s6','%s7','%s8','%s9')" % (w,w,w,w,w,w,w,w,w,w))
+
+  try:
+    db = dbms.get(user, get_query_server_config())
+    db.use(table_info['db'])
+    query = hql_query(drop_sql)
+    handle = db.execute_and_wait(query, timeout_sec=120)
+    query = hql_query(create_sql)
+    handle = db.execute_and_wait(query, timeout_sec=120)
+    query = hql_query(hql.getvalue())
+    handle = db.execute_and_wait(query, timeout_sec=300)
+    hql.close()
+  except Exception, ex:
+    failed = True
+
+  # Big table creation (data upload) is successful
+  assert_false(failed)
+
+  # Fetch large data set
+  hql = "SELECT w0,w1,w2,w3,w4,w5,w6,w7,w8,w9,w0,w1,w2,w3,w4,w5,w6,w7,w8,w9 FROM %(db)s.%(table_name)s" % table_info
+
+  # large rows
+  max_rows = 8745
 
+  try:
+    query = hql_query(hql)
+    handle = db.execute_and_wait(query)
+    results = db.fetch(handle, True, max_rows-20)
+  except QueryServerException, ex:
+    if 'Invalid OperationHandle' in ex.message and 'EXECUTE_STATEMENT' in ex.message:
+      failed = True
+  except:
+      failed = True
+
+  # Fetch large data set is successful because SASL_MAX_BUFFER > RESULT_DATA
+  assert_false(failed)
+
+  # Test case when SASL_MAX_BUFFER < RESULT_DATA
+  try:
+    query = hql_query(hql)
+    handle = db.execute_and_wait(query)
+    results = db.fetch(handle, True, max_rows)
+  except QueryServerException, ex:
+    if 'Invalid OperationHandle' in ex.message and 'EXECUTE_STATEMENT' in ex.message:
+      failed = True
+  except:
+      failed = True
+
+  # Fetch large data set fails because SASL_MAX_BUFFER < RESULT_DATA In your log file you will see following log lines
+  # thrift_util  INFO     Thrift exception; retrying: Error in sasl_decode (-1) SASL(-1): generic failure: Unable to find a callback: 32775
+  # thrift_util  INFO     Increase the SASL_MAX_BUFFER value in hue.ini
+  assert_true(failed)
+  failed = False
+
+  # Cleanup
+  hql = "DROP TABLE %(db)s.%(table_name)s" % table_info
 
+  try:
+    query = hql_query(hql)
+    handle = db.execute_and_wait(query)
+  except QueryServerException, ex:
+    if 'Invalid OperationHandle' in ex.message and 'EXECUTE_STATEMENT' in ex.message:
+      failed = True
+  except:
+      failed = True
+  assert_false(failed)

+ 3 - 0
desktop/conf.dist/hue.ini

@@ -69,6 +69,9 @@
   # Number of threads used by the CherryPy web server
   ## cherrypy_server_threads=40
 
+  # This property specifies the maximum size of the receive buffer in bytes in thrift sasl communication.
+  ## sasl_max_buffer=2*1024*1024  # 2MB
+
   # Filename of SSL Certificate
   ## ssl_certificate=
 

+ 3 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -73,6 +73,9 @@
   # Number of threads used by the CherryPy web server
   ## cherrypy_server_threads=40
 
+  # This property specifies the maximum size of the receive buffer in bytes in thrift sasl communication.
+  ## sasl_max_buffer=2*1024*1024  # 2MB
+
   # Filename of SSL Certificate
   ## ssl_certificate=
 

+ 7 - 0
desktop/core/src/desktop/conf.py

@@ -654,6 +654,13 @@ KERBEROS = ConfigSection(
   )
 )
 
+SASL_MAX_BUFFER = Config(
+  key="sasl_max_buffer",
+  help=_("This property specifies the maximum size of the receive buffer in bytes in thrift sasl communication."),
+  default=2*1024*1024,  # 2 MB
+  type=int
+)
+
 # See python's documentation for time.tzset for valid values.
 TIME_ZONE = Config(
   key="time_zone",

+ 6 - 0
desktop/core/src/desktop/lib/thrift_util.py

@@ -32,6 +32,8 @@ from thrift.protocol.TBinaryProtocol import TBinaryProtocol
 from thrift.protocol.TMultiplexedProtocol import TMultiplexedProtocol
 
 from django.conf import settings
+from desktop.conf import SASL_MAX_BUFFER
+
 from desktop.lib.python_util import create_synchronous_io_multiplexer
 from desktop.lib.thrift_.http_client import THttpClient
 from desktop.lib.thrift_.TSSLSocketWithWildcardSAN import TSSLSocketWithWildcardSAN
@@ -283,6 +285,8 @@ def connect_to_thrift(conf):
       if conf.mechanism == 'PLAIN':
         saslc.setAttr("username", str(conf.username))
         saslc.setAttr("password", str(conf.password)) # Defaults to 'hue' for a non-empty string unless using LDAP
+      else:
+        saslc.setAttr("maxbufsize", SASL_MAX_BUFFER.get())
       saslc.init()
       return saslc
     transport = TSaslClientTransport(sasl_factory, conf.mechanism, mode)
@@ -459,6 +463,8 @@ class SuperClient(object):
           tries_left -= 1
           if tries_left:
             logging.info("Thrift exception; retrying: " + str(e), exc_info=0)
+            if 'generic failure: Unable to find a callback: 32775' in str(e):
+              logging.warn("Increase the SASL_MAX_BUFFER value in hue.ini")
       logging.warn("Out of retries for thrift call: " + attr)
       raise
     return wrapper