فهرست منبع

HUE-1075 [beeswax] Support Hive Server 2

Romain Rigaux 12 سال پیش
والد
کامیت
1162e67c79

+ 13 - 3
apps/beeswax/gen-py/cli_service/TCLIService-remote → apps/beeswax/gen-py/TCLIService/TCLIService-remote

@@ -1,9 +1,11 @@
 #!/usr/bin/env python
 #
-# Autogenerated by Thrift Compiler (0.7.0)
+# Autogenerated by Thrift Compiler (0.9.0)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
+#  options string: py:new_style
+#
 
 import sys
 import pprint
@@ -18,7 +20,7 @@ from ttypes import *
 
 if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print ''
-  print 'Usage: ' + sys.argv[0] + ' [-h host:port] [-u url] [-f[ramed]] function [arg1 [arg2...]]'
+  print 'Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] function [arg1 [arg2...]]'
   print ''
   print 'Functions:'
   print '  TOpenSessionResp OpenSession(TOpenSessionReq req)'
@@ -37,6 +39,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print '  TCloseOperationResp CloseOperation(TCloseOperationReq req)'
   print '  TGetResultSetMetadataResp GetResultSetMetadata(TGetResultSetMetadataReq req)'
   print '  TFetchResultsResp FetchResults(TFetchResultsReq req)'
+  print '  TGetLogResp GetLog(TGetLogReq req)'
   print ''
   sys.exit(0)
 
@@ -51,7 +54,8 @@ argi = 1
 if sys.argv[argi] == '-h':
   parts = sys.argv[argi+1].split(':')
   host = parts[0]
-  port = int(parts[1])
+  if len(parts) > 1:
+    port = int(parts[1])
   argi += 2
 
 if sys.argv[argi] == '-u':
@@ -183,6 +187,12 @@ elif cmd == 'FetchResults':
     sys.exit(1)
   pp.pprint(client.FetchResults(eval(args[0]),))
 
+elif cmd == 'GetLog':
+  if len(args) != 1:
+    print 'GetLog requires 1 args'
+    sys.exit(1)
+  pp.pprint(client.GetLog(eval(args[0]),))
+
 else:
   print 'Unrecognized method %s' % cmd
   sys.exit(1)

تفاوت فایلی نمایش داده نمی شود زیرا این فایل بسیار بزرگ است
+ 171 - 121
apps/beeswax/gen-py/TCLIService/TCLIService.py


+ 0 - 0
apps/beeswax/gen-py/cli_service/__init__.py → apps/beeswax/gen-py/TCLIService/__init__.py


+ 6 - 2
apps/beeswax/gen-py/cli_service/constants.py → apps/beeswax/gen-py/TCLIService/constants.py

@@ -1,10 +1,12 @@
 #
-# Autogenerated by Thrift Compiler (0.7.0)
+# Autogenerated by Thrift Compiler (0.9.0)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
+#  options string: py:new_style
+#
 
-from thrift.Thrift import *
+from thrift.Thrift import TType, TMessageType, TException, TApplicationException
 from ttypes import *
 
 PRIMITIVE_TYPES = set([
@@ -18,6 +20,7 @@ PRIMITIVE_TYPES = set([
     7,
     8,
     9,
+    15,
 ])
 COMPLEX_TYPES = set([
     10,
@@ -45,4 +48,5 @@ TYPE_NAMES = {
     11 : "MAP",
     12 : "STRUCT",
     13 : "UNIONTYPE",
+    15 : "DECIMAL",
 }

تفاوت فایلی نمایش داده نمی شود زیرا این فایل بسیار بزرگ است
+ 164 - 160
apps/beeswax/gen-py/TCLIService/ttypes.py


+ 1 - 1
apps/beeswax/regenerate_thrift.sh

@@ -22,7 +22,7 @@ cd $(dirname $0)
 
 thrift -I thrift/include -r --gen py:new_style -o ./ thrift/beeswax.thrift
 thrift -I thrift/include -r --gen java:hashcode -o java/src/main thrift/beeswax.thrift
-thrift -I thrift/include -r --gen py:new_style -o ./ thrift/cli_service.thrift
+thrift -I thrift/include -r --gen py:new_style -o ./ thrift/TCLIService.thrift
 
 # We don't need to have generated code for the metastore, since that's
 # in one of the hive jars that we include

+ 1 - 1
apps/beeswax/setup.py

@@ -23,7 +23,7 @@ setup(
       url = 'http://github.com/cloudera/hue',
       description = "Hive Interface on Hue",
       packages = find_packages('src') + find_packages('gen-py'),
-      package_dir = {'hive_metastore': 'gen-py', 'beeswaxd': 'gen-py', 'fb303': 'gen-py', 'cli_service': 'gen-py', '': 'src'},
+      package_dir = {'hive_metastore': 'gen-py', 'beeswaxd': 'gen-py', 'fb303': 'gen-py', 'TCLIService': 'gen-py', '': 'src'},
       install_requires = ['setuptools', 'desktop'],
       entry_points = {
         'desktop.sdk.application': 'beeswax=beeswax',

+ 2 - 3
apps/beeswax/src/beeswax/forms.py

@@ -25,7 +25,6 @@ from desktop.lib.django_forms import ChoiceOrOtherField, MultiForm, SubmitButton
 from filebrowser.forms import PathField
 
 from beeswax import common
-from beeswax.server.dbms import NoSuchObjectException
 from beeswax.models import SavedQuery
 
 
@@ -126,7 +125,7 @@ class SaveResultsForm(DependencyAwareForm):
             self.db.get_table('default', tbl) # Assumes 'default' DB
           self._errors['target_table'] = self.error_class([_('Table already exists')])
           del cleaned_data['target_table']
-        except hive_metastore.ttypes.NoSuchObjectException:
+        except Exception:
           pass
 
     return cleaned_data
@@ -260,7 +259,7 @@ def _clean_tablename(db, name):
     table = db.get_table("default", name)
     if table.name:
       raise forms.ValidationError(_('Table "%(name)s" already exists') % {'name': name})
-  except (hive_metastore.ttypes.NoSuchObjectException, NoSuchObjectException):
+  except Exception:
     return name
 
 

+ 2 - 1
apps/beeswax/src/beeswax/models.py

@@ -32,7 +32,7 @@ from desktop.lib.exceptions_renderable import PopupException
 from beeswax.conf import SERVER_INTERFACE
 from beeswax.design import HQLdesign, hql_query
 from beeswaxd.ttypes import QueryHandle as BeeswaxdQueryHandle, QueryState
-from cli_service.ttypes import TSessionHandle, THandleIdentifier,\
+from TCLIService.ttypes import TSessionHandle, THandleIdentifier,\
   TOperationState, TOperationHandle, TOperationType
 
 
@@ -183,6 +183,7 @@ class HiveServerQueryHistory(QueryHistory):
 
   def save_state(self, new_state):
     self.last_state = new_state.index
+    self.save()
 
 
 class BeeswaxQueryHistory(QueryHistory):

+ 15 - 5
apps/beeswax/src/beeswax/server/dbms.py

@@ -73,7 +73,12 @@ def get_query_server_config(name='beeswax', requires_ddl=False):
   return query_server
 
 
-class QueryServerException: pass
+class QueryServerException(Exception):
+  # Ideally the query handle will be stored here too.
+
+  def __init__(self, e):
+    super(QueryServerException, self).__init__(e)
+
 
 class NoSuchObjectException: pass
 
@@ -332,12 +337,17 @@ class DataTable:
 
 
 # TODO decorator?
-def expand_exception(exc, db):
+def expand_exception(exc, db, handle=None):
   try:
-    log = db.get_log(exc)
-  except:
+    if handle is not None:
+      log = db.get_log(handle)
+    elif hasattr(handle, 'get_rpc_handle'):
+      log = db.get_log(exc)
+    else:
+      log = _("No server logs for this query")
+  except Exception, e:
     # Always show something, even if server has died on the job.
-    log = _("Could not retrieve logs.")
+    log = _("Could not retrieve logs: %s" % e)
 
   if not exc.message:
     error_message = _("Unknown exception.")

+ 91 - 52
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -15,20 +15,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+import logging
 import thrift
 
 from desktop.lib import thrift_util
 
-from cli_service import TCLIService
-from cli_service.ttypes import TOpenSessionReq, TGetTablesReq, TFetchResultsReq,\
+from TCLIService import TCLIService
+from TCLIService.ttypes import TOpenSessionReq, TGetTablesReq, TFetchResultsReq,\
   TStatusCode, TGetResultSetMetadataReq, TGetColumnsReq, TType,\
   TExecuteStatementReq, TGetOperationStatusReq, TFetchOrientation,\
-  TCloseSessionReq, TGetCatalogsReq
+  TCloseSessionReq, TGetSchemasReq, TGetLogReq
 
 from beeswax import conf
 from beeswax.models import Session, HiveServerQueryHandle, HiveServerQueryHistory
-from beeswax.server.dbms import Table, NoSuchObjectException, DataTable
+from beeswax.server.dbms import Table, NoSuchObjectException, DataTable,\
+  QueryServerException
+
+
+LOG = logging.getLogger(__name__)
 
 
 class HiveServerTable(Table):
@@ -235,45 +239,51 @@ class HiveServerClient:
                                           timeout_seconds=conf.BEESWAX_SERVER_CONN_TIMEOUT.get())
 
 
+  def open_session(self, user):
+    req = TOpenSessionReq(username=user.username, configuration={})
+    res = self._client.OpenSession(req)
+
+    sessionId = res.sessionHandle.sessionId
+    LOG.info('Opening session %s' % sessionId)
+
+    encoded_status, encoded_guid = HiveServerQueryHandle(secret=sessionId.secret, guid=sessionId.guid).get()
+
+    return Session.objects.create(owner=user,
+                                  status_code=res.status.statusCode,
+                                  secret=encoded_status,
+                                  guid=encoded_guid,
+                                  server_protocol_version=res.serverProtocolVersion)
+
+
   def call(self, fn, req, status=TStatusCode.SUCCESS_STATUS):
     session = Session.objects.get_session(self.user)
+
     if session is None:
       session = self.open_session(self.user)
+
     if hasattr(req, 'sessionHandle') and req.sessionHandle is None:
       req.sessionHandle = session.get_handle()
 
     res = fn(req)
 
-    if res.status.statusCode == TStatusCode.ERROR_STATUS: #TODO should be TStatusCode.INVALID_HANDLE_STATUS
-      print 'HS2 Session has expired: %s' % res
-      print 'Retrying with a new sessions...'
+    # Not supported currently in HS2: TStatusCode.INVALID_HANDLE_STATUS
+    if res.status.statusCode == TStatusCode.ERROR_STATUS and \
+        res.status.errorMessage is not None and 'Invalid SessionHandle' in res.status.errorMessage:
+      LOG.info('Retrying with a new session because of %s' % res)
 
       session = self.open_session(self.user)
       req.sessionHandle = session.get_handle()
 
-      res = fn(req)
+      # Get back the name of the function to call
+      res = getattr(self._client, fn.attr)(req)
 
     if status is not None and res.status.statusCode not in (
         TStatusCode.SUCCESS_STATUS, TStatusCode.SUCCESS_WITH_INFO_STATUS, TStatusCode.STILL_EXECUTING_STATUS):
-      raise Exception('Bad status for request %s:\n%s' % (req, res))
+      raise QueryServerException(Exception('Bad status for request %s:\n%s' % (req, res)))
     else:
       return res
 
 
-  def open_session(self, user):
-    req = TOpenSessionReq(username=user.username)
-    res = self._client.OpenSession(req)
-
-    sessionId = res.sessionHandle.sessionId
-
-    encoded_status, encoded_guid = HiveServerQueryHandle(secret=sessionId.secret, guid=sessionId.guid).get()
-
-    return Session.objects.create(owner=user,
-                                  status_code=res.status.statusCode,
-                                  secret=encoded_status,
-                                  guid=encoded_guid,
-                                  server_protocol_version=res.serverProtocolVersion)
-
   def close_session(self):
     session = Session.objects.get_session(self.user).get_handle()
 
@@ -282,12 +292,13 @@ class HiveServerClient:
 
 
   def get_databases(self):
-    req = TGetCatalogsReq()
-    res = self.call(self._client.GetCatalogs, req)
+    # GetCatalogs() is not implemented in HS2
+    req = TGetSchemasReq()
+    res = self.call(self._client.GetSchemas, req)
 
     results, schema = self.fetch_result(res.operationHandle)
 
-    return HiveServerTRowSet(results.results, schema.schema).cols(('TABLE_CATALOG',))
+    return HiveServerTRowSet(results.results, schema.schema).cols(('TABLE_SCHEMA',))
 
 
   def get_tables(self, database, table_names):
@@ -300,8 +311,9 @@ class HiveServerClient:
 
 
   def get_table(self, database, table_name):
-    req = TGetTablesReq(schemaName='default', tableName=table_name)
+    req = TGetTablesReq(schemaName=database, tableName=table_name)
     res = self.call(self._client.GetTables, req)
+
     table_results, table_schema = self.fetch_result(res.operationHandle)
 
     # Using 'SELECT * from table' does not show column comments in the metadata
@@ -309,26 +321,31 @@ class HiveServerClient:
     return HiveServerTable(table_results.results, table_schema.schema, desc_results.results, desc_schema.schema)
 
 
-  def execute_query(self, query, max_rows=1000):
-    # TODO: Need jars, UDF etc
-    results, schema = self.execute_statement(statement=query.query['query'], conf_overlay={}, max_rows=max_rows)
+  def execute_query(self, query, max_rows=100):
+    # TODO: Need to set jars, UDF etc
+    self.execute_statement(statement='SET hive.server2.blocking.query=true')
+
+    results, schema = self.execute_statement(statement=query.query['query'], max_rows=max_rows)
     return HiveServerDataTable(results, schema)
 
 
   def execute_async_query(self, query, statement=0):
+    self.execute_statement(statement='SET hive.server2.blocking.query=false')
+
     query_statement = query.get_query_statement(statement)
-    return self.execute_async_statement(statement=query_statement, conf_overlay={})
+    return self.execute_async_statement(statement=query_statement)
 
 
-  def execute_statement(self, statement, conf_overlay=None, max_rows=100):
-    req = TExecuteStatementReq(statement=statement)
+  def execute_statement(self, statement, max_rows=100):
+    req = TExecuteStatementReq(statement=statement, confOverlay={})
     res = self.call(self._client.ExecuteStatement, req)
 
     return self.fetch_result(res.operationHandle, max_rows=max_rows)
 
 
-  def execute_async_statement(self, statement, conf_overlay=None):
-    req = TExecuteStatementReq(statement=statement)
+  def execute_async_statement(self, statement):
+    # confOverlay is not used by Hive Server 2
+    req = TExecuteStatementReq(statement=statement, confOverlay={})
     res = self.call(self._client.ExecuteStatement, req)
 
     return HiveServerQueryHandle(secret=res.operationHandle.operationId.secret,
@@ -338,20 +355,20 @@ class HiveServerClient:
                                  modified_row_count=res.operationHandle.modifiedRowCount)
 
 
-  def fetch_data(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=10000):
-    # The client should check for hasMoreRows and fetch until the result is empty
+  def fetch_data(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=100):
+    # The client should check for hasMoreRows and fetch until the result is empty dues to a HS2 bug
     results, schema = self.fetch_result(operation_handle, orientation, max_rows)
     return HiveServerDataTable(results, schema)
 
 
-  def get_columns(self, table):
-    req = TGetColumnsReq(schemaName='default', tableName=table)
+  def get_columns(self, database, table):
+    req = TGetColumnsReq(schemaName=database, tableName=table)
     res = self.call(self._client.GetColumns, req)
 
     return self.fetch_result(res.operationHandle)
 
 
-  def fetch_result(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=10000):
+  def fetch_result(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=100):
     fetch_req = TFetchResultsReq(operationHandle=operation_handle, orientation=orientation, maxRows=max_rows)
     res = self.call(self._client.FetchResults, fetch_req)
 
@@ -366,9 +383,14 @@ class HiveServerClient:
 
   def get_operation_status(self, operation_handle):
     req = TGetOperationStatusReq(operationHandle=operation_handle)
-    res = self.call(self._client.GetOperationStatus, req)
+    return self.call(self._client.GetOperationStatus, req)
 
-    return res
+
+  def get_log(self, operation_handle):
+    req = TGetLogReq(operationHandle=operation_handle)
+    res = self.call(self._client.GetLog, req)
+
+    return res.log
 
 
 class HiveServerTableCompatible(HiveServerTable):
@@ -387,6 +409,24 @@ class HiveServerTableCompatible(HiveServerTable):
                                     'comment': col.get('comment', ''), }) for col in HiveServerTable.cols.fget(self)]
 
 
+class ResultCompatible:
+
+  def __init__(self, data_table):
+    self.data_table = data_table
+    self.rows = data_table.rows
+    self.has_more = data_table.has_more
+    self.start_row = data_table.startRowOffset
+    self.ready = True
+
+  @property
+  def columns(self):
+    return self.cols()
+
+
+  def cols(self):
+    return [col.name for col in self.data_table.cols()]
+
+
 class HiveServerClientCompatible:
   """Same API as Beeswax"""
 
@@ -402,8 +442,8 @@ class HiveServerClientCompatible:
 
   def get_state(self, handle):
     operationHandle = handle.get_rpc_handle()
-
     res = self._client.get_operation_status(operationHandle)
+
     return HiveServerQueryHistory.STATE_MAP[res.operationState]
 
 
@@ -423,10 +463,8 @@ class HiveServerClientCompatible:
 
     data_table = self._client.fetch_data(operationHandle, orientation=orientation, max_rows=max_rows)
 
-    return type('Result', (object,), {'rows': data_table.rows,
-                                      'columns': [col.name for col in data_table.cols()],
-                                      'has_more': data_table.has_more,
-                                      'start_row': data_table.startRowOffset, })
+    return ResultCompatible(data_table)
+
 
   def dump_config(self):
     return 'Does not exist in HS2'
@@ -436,12 +474,13 @@ class HiveServerClientCompatible:
     return 'Does not exist in HS2'
 
 
-  def get_log(self, *args, **kwargs):
-    return 'No logs retrieval implemented'
+  def get_log(self, handle):
+    operationHandle = handle.get_rpc_handle()
+    return self._client.get_log(operationHandle)
 
 
-  def get_databases(self, database, table_names):
-    return [table['TABLE_CAT'] for table in self._client.get_databases()]
+  def get_databases(self):
+    return [table['TABLE_SCHEMA'] for table in self._client.get_databases()]
 
 
   def get_tables(self, database, table_names):

+ 12 - 15
apps/beeswax/src/beeswax/views.py

@@ -38,7 +38,6 @@ from desktop.lib.exceptions_renderable import PopupException
 
 from hadoop.fs.exceptions import WebHdfsException
 from jobsub.parameterization import find_variables, substitute_variables
-from beeswaxd.ttypes import BeeswaxException, QueryNotFoundException
 
 import beeswax.forms
 import beeswax.design
@@ -323,7 +322,7 @@ def describe_table(request, database, table):
 
   try:
     table_data = db.get_sample(database, table)
-  except BeeswaxException, ex:
+  except Exception, ex:
     error_message, logs = expand_exception(ex, db)
 
   return render("describe_table.mako", request, {
@@ -347,7 +346,7 @@ def drop_table(request, database):
       query_history = db.drop_tables(database, tables_objects, design)
       url = reverse(app_name + ':watch_query', args=[query_history.id]) + '?on_success_url=' + reverse(app_name + ':show_tables')
       return redirect(url)
-    except BeeswaxException, ex:
+    except Exception, ex:
       error_message, log = expand_exception(ex, db)
       error = _("Failed to remove %(tables)s.  Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message}
       raise PopupException(error, title=_("Beeswax Error"), detail=log)
@@ -494,9 +493,7 @@ def execute_query(request, design_id=None):
           else:
             download = request.POST.has_key('download')
             return execute_directly(request, query, query_server, design, on_success_url=on_success_url, download=download)
-        except BeeswaxException, ex:
-          print ex.errorCode
-          print ex.SQLState
+        except Exception, ex:
           error_message, log = expand_exception(ex, db)
   else:
     if design.id is not None:
@@ -562,7 +559,7 @@ def watch_query(request, id):
   if request.method == 'POST' or (not query_history.is_finished() and query_history.is_success() and not query_history.has_results):
     try:
       query_history = db.execute_next_statement(query_history)
-    except BeeswaxException, ex:
+    except Exception, ex:
       pass
 
   # Check query state
@@ -570,7 +567,7 @@ def watch_query(request, id):
   query_history.save_state(state)
 
   if query_history.is_failure():
-    # When we fetch, Beeswax server will throw us a BeeswaxException, which has the
+    # When we fetch, Beeswax server will throw us a Exception, which has the
     # log we want to display.
     return format_preserving_redirect(request, results_url, request.GET)
   elif query_history.is_finished() or (query_history.is_success() and query_history.has_results):
@@ -601,13 +598,13 @@ def watch_query_refresh_json(request, id):
     if not query_history.is_finished() and query_history.is_success() and not query_history.has_results:
       db.execute_next_statement(query_history)
       handle, state = _get_query_handle_and_state(query_history)
-  except BeeswaxException, ex:
+  except Exception, ex:
     LOG.exception(ex)
     handle, state = _get_query_handle_and_state(query_history)
 
   try:
     log = db.get_log(handle)
-  except BeeswaxException, ex:
+  except Exception, ex:
     log = str(ex)
 
   jobs = _parse_out_hadoop_jobs(log)
@@ -682,7 +679,7 @@ def view_results(request, id, first_row=0):
     log = db.get_log(handle)
   except Exception, ex:
     fetch_error = True
-    error_message, log = expand_exception(ex, db)
+    error_message, log = expand_exception(ex, db, handle)
 
   # Handle errors
   error = fetch_error or results is None or expired
@@ -761,7 +758,7 @@ def save_results(request, id):
       try:
         handle, state = _get_query_handle_and_state(query_history)
         result_meta = db.get_results_metadata(handle)
-      except QueryNotFoundException, ex:
+      except Exception, ex:
         LOG.exception(ex)
         raise PopupException(_('Cannot find query.'))
       if result_meta.table_dir:
@@ -787,7 +784,7 @@ def save_results(request, id):
           # To new table
           try:
             return _save_results_ctas(request, query_history, form.cleaned_data['target_table'], result_meta)
-          except BeeswaxException, bex:
+          except Exception, bex:
             LOG.exception(bex)
             error_msg, log = expand_exception(bex, db)
       except WebHdfsException, ex:
@@ -811,7 +808,7 @@ def save_results(request, id):
 def _save_results_ctas(request, query_history, target_table, result_meta):
   """
   Handle saving results as a new table. Returns HTTP response.
-  May raise BeeswaxException, IOError.
+  May raise Exception, IOError.
   """
   query_server = query_history.get_query_server_config() # Query server requires DDL support
   db = dbms.get(request.user)
@@ -1109,7 +1106,7 @@ def _run_parameterized_query(request, design_id, explain):
         return explain_directly(request, query, design, query_server)
       else:
         return execute_directly(request, query, query_server, design)
-    except BeeswaxException, ex:
+    except Exception, ex:
       db = dbms.get(request.user, query_server)
       error_message, log = expand_exception(ex, db)
       return render('execute.mako', request, {

+ 87 - 63
apps/beeswax/thrift/cli_service.thrift → apps/beeswax/thrift/TCLIService.thrift

@@ -16,20 +16,21 @@
 
 // Coding Conventions for this file:
 //
-// Structs and Unions
-// * Struct and union names begin with a "T", and use a capital letter for
-//   each new word, with no underscores.
+// Structs/Enums/Unions
+// * Struct, Enum, and Union names begin with a "T",
+//   and use a capital letter for each new word, with no underscores.
 // * All fields should be declared as either optional or required.
 //
 // Functions
 // * Function names start with a capital letter and have a capital letter for
 //   each new word, with no underscores.
-// * Each function should take exactly one parameter, named FunctionNameReq,
-//   and should return either void or FunctionNameResp. This convention allows
+// * Each function should take exactly one parameter, named TFunctionNameReq,
+//   and should return either void or TFunctionNameResp. This convention allows
 //   incremental updates.
 //
 // Services
-// * Service names should end in the word "Service".
+// * Service names begin with the letter "T", use a capital letter for each
+//   new word (with no underscores), and end with the word "Service".
 
 namespace java org.apache.hive.service.cli.thrift
 namespace cpp apache.hive.service.cli.thrift
@@ -40,7 +41,7 @@ enum TProtocolVersion {
   HIVE_CLI_SERVICE_PROTOCOL_V1
 }
 
-enum TType {
+enum TTypeId {
   BOOLEAN_TYPE,
   TINYINT_TYPE,
   SMALLINT_TYPE,
@@ -55,50 +56,53 @@ enum TType {
   MAP_TYPE,
   STRUCT_TYPE,
   UNION_TYPE,
-  USER_DEFINED_TYPE
-}
-  
-const set<TType> PRIMITIVE_TYPES = [
-  TType.BOOLEAN_TYPE
-  TType.TINYINT_TYPE
-  TType.SMALLINT_TYPE
-  TType.INT_TYPE
-  TType.BIGINT_TYPE
-  TType.FLOAT_TYPE
-  TType.DOUBLE_TYPE
-  TType.STRING_TYPE
-  TType.TIMESTAMP_TYPE
-  TType.BINARY_TYPE
+  USER_DEFINED_TYPE,
+  DECIMAL_TYPE
+}
+
+const set<TTypeId> PRIMITIVE_TYPES = [
+  TTypeId.BOOLEAN_TYPE
+  TTypeId.TINYINT_TYPE
+  TTypeId.SMALLINT_TYPE
+  TTypeId.INT_TYPE
+  TTypeId.BIGINT_TYPE
+  TTypeId.FLOAT_TYPE
+  TTypeId.DOUBLE_TYPE
+  TTypeId.STRING_TYPE
+  TTypeId.TIMESTAMP_TYPE
+  TTypeId.BINARY_TYPE,
+  TTypeId.DECIMAL_TYPE
 ]
 
-const set<TType> COMPLEX_TYPES = [
-  TType.ARRAY_TYPE
-  TType.MAP_TYPE
-  TType.STRUCT_TYPE
-  TType.UNION_TYPE
-  TType.USER_DEFINED_TYPE
+const set<TTypeId> COMPLEX_TYPES = [
+  TTypeId.ARRAY_TYPE
+  TTypeId.MAP_TYPE
+  TTypeId.STRUCT_TYPE
+  TTypeId.UNION_TYPE
+  TTypeId.USER_DEFINED_TYPE
 ]
 
-const set<TType> COLLECTION_TYPES = [
-  TType.ARRAY_TYPE
-  TType.MAP_TYPE
+const set<TTypeId> COLLECTION_TYPES = [
+  TTypeId.ARRAY_TYPE
+  TTypeId.MAP_TYPE
 ]
 
-const map<TType,string> TYPE_NAMES = {
-  TType.BOOLEAN_TYPE: "BOOLEAN",
-  TType.TINYINT_TYPE: "TINYINT",
-  TType.SMALLINT_TYPE: "SMALLINT",
-  TType.INT_TYPE: "INT",
-  TType.BIGINT_TYPE: "BIGINT",
-  TType.FLOAT_TYPE: "FLOAT",
-  TType.DOUBLE_TYPE: "DOUBLE",
-  TType.STRING_TYPE: "STRING",
-  TType.TIMESTAMP_TYPE: "TIMESTAMP",
-  TType.BINARY_TYPE: "BINARY",
-  TType.ARRAY_TYPE: "ARRAY",
-  TType.MAP_TYPE: "MAP",
-  TType.STRUCT_TYPE: "STRUCT",
-  TType.UNION_TYPE: "UNIONTYPE"
+const map<TTypeId,string> TYPE_NAMES = {
+  TTypeId.BOOLEAN_TYPE: "BOOLEAN",
+  TTypeId.TINYINT_TYPE: "TINYINT",
+  TTypeId.SMALLINT_TYPE: "SMALLINT",
+  TTypeId.INT_TYPE: "INT",
+  TTypeId.BIGINT_TYPE: "BIGINT",
+  TTypeId.FLOAT_TYPE: "FLOAT",
+  TTypeId.DOUBLE_TYPE: "DOUBLE",
+  TTypeId.STRING_TYPE: "STRING",
+  TTypeId.TIMESTAMP_TYPE: "TIMESTAMP",
+  TTypeId.BINARY_TYPE: "BINARY",
+  TTypeId.ARRAY_TYPE: "ARRAY",
+  TTypeId.MAP_TYPE: "MAP",
+  TTypeId.STRUCT_TYPE: "STRUCT",
+  TTypeId.UNION_TYPE: "UNIONTYPE"
+  TTypeId.DECIMAL_TYPE: "DECIMAL"
 }
 
 // Thrift does not support recursively defined types or forward declarations,
@@ -150,7 +154,7 @@ typedef i32 TTypeEntryPtr
 struct TPrimitiveTypeEntry {
   // The primitive type token. This must satisfy the condition
   // that type is in the PRIMITIVE_TYPES set.
-  1: required TType type
+  1: required TTypeId type
 }
 
 // Type entry for an ARRAY type.
@@ -204,7 +208,7 @@ struct TColumnDesc {
 
   // The type descriptor for this column
   2: required TTypeDesc typeDesc
-  
+
   // The ordinal position of this column in the schema
   3: required i32 position
 
@@ -254,6 +258,7 @@ struct TDoubleValue {
 }
 
 struct TStringValue {
+  // NULL if value is unset
   1: optional string value
 }
 
@@ -280,7 +285,7 @@ union TColumnValue {
   4: TI32Value    i32Val       // INT
   5: TI64Value    i64Val       // BIGINT, TIMESTAMP
   6: TDoubleValue doubleVal    // FLOAT, DOUBLE
-  7: TStringValue stringVal    // STRING, LIST, MAP, STRUCT, UNIONTYPE, BINARY
+  7: TStringValue stringVal    // STRING, LIST, MAP, STRUCT, UNIONTYPE, BINARY, DECIMAL
 }
 
 // Represents a row in a rowset.
@@ -438,11 +443,11 @@ struct TOperationHandle {
 // OpenSession()
 //
 // Open a session (connection) on the server against
-// which operations may be executed. 
+// which operations may be executed.
 struct TOpenSessionReq {
   // The version of the HiveServer2 protocol that the client is using.
   1: required TProtocolVersion client_protocol = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V1
-  
+
   // Username and password for authentication.
   // Depending on the authentication scheme being used,
   // this information may instead be provided by a lower
@@ -606,13 +611,13 @@ struct TGetTypeInfoReq {
 struct TGetTypeInfoResp {
   1: required TStatus status
   2: optional TOperationHandle operationHandle
-}  
+}
 
 
 // GetCatalogs()
 //
-// Returns the list of catalogs (databases) 
-// Results are ordered by TABLE_CATALOG 
+// Returns the list of catalogs (databases)
+// Results are ordered by TABLE_CATALOG
 //
 // Resultset columns :
 // col1
@@ -633,7 +638,7 @@ struct TGetCatalogsResp {
 
 // GetSchemas()
 //
-// Retrieves the schema names available in this database. 
+// Retrieves the schema names available in this database.
 // The results are ordered by TABLE_CATALOG and TABLE_SCHEM.
 // col1
 // name: TABLE_SCHEM
@@ -722,9 +727,9 @@ struct TGetTablesResp {
 
 // GetTableTypes()
 //
-// Returns the table types available in this database. 
-// The results are ordered by table type. 
-// 
+// Returns the table types available in this database.
+// The results are ordered by table type.
+//
 // col1
 // name: TABLE_TYPE
 // type: STRING
@@ -745,8 +750,8 @@ struct TGetTableTypesResp {
 // Returns a list of columns in the specified tables.
 // The information is returned as a result set which can be fetched
 // using the OperationHandle provided in the response.
-// Results are ordered by TABLE_CAT, TABLE_SCHEM, TABLE_NAME, 
-// and ORDINAL_POSITION. 
+// Results are ordered by TABLE_CAT, TABLE_SCHEM, TABLE_NAME,
+// and ORDINAL_POSITION.
 //
 // Result Set Columns are the same as those for the ODBC CLIColumns
 // function.
@@ -842,7 +847,7 @@ struct TGetFunctionsResp {
   1: required TStatus status
   2: optional TOperationHandle operationHandle
 }
-  
+
 
 // GetOperationStatus()
 //
@@ -937,7 +942,7 @@ struct TFetchResultsReq {
   // The fetch orientation. For V1 this must be either
   // FETCH_NEXT or FETCH_FIRST. Defaults to FETCH_NEXT.
   2: required TFetchOrientation orientation = TFetchOrientation.FETCH_NEXT
-  
+
   // Max number of rows that should be returned in
   // the rowset.
   3: required i64 maxRows
@@ -956,6 +961,22 @@ struct TFetchResultsResp {
   3: optional TRowSet results
 }
 
+// GetLog()
+//
+// Fetch operation log from the server corresponding to
+// a particular OperationHandle.
+struct TGetLogReq {
+  // Operation whose log is requested
+  1: required TOperationHandle operationHandle
+}
+
+struct TGetLogResp {
+  1: required TStatus status
+
+  2: required string log
+
+}
+
 service TCLIService {
 
   TOpenSessionResp OpenSession(1:TOpenSessionReq req);
@@ -981,12 +1002,15 @@ service TCLIService {
   TGetFunctionsResp GetFunctions(1:TGetFunctionsReq req);
 
   TGetOperationStatusResp GetOperationStatus(1:TGetOperationStatusReq req);
-  
+
   TCancelOperationResp CancelOperation(1:TCancelOperationReq req);
 
   TCloseOperationResp CloseOperation(1:TCloseOperationReq req);
 
   TGetResultSetMetadataResp GetResultSetMetadata(1:TGetResultSetMetadataReq req);
-  
+
   TFetchResultsResp FetchResults(1:TFetchResultsReq req);
+
+  TGetLogResp GetLog(1:TGetLogReq req);
 }
+

+ 11 - 8
desktop/core/src/desktop/lib/thrift_util.py

@@ -90,6 +90,8 @@ class ConnectionConfig(object):
     self.kerberos_principal = kerberos_principal
     self.timeout_seconds = timeout_seconds
 
+  def __str__(self):
+    return ', '.join(map(str, [self.klass, self.host, self.port, self.service_name, self.use_sasl, self.kerberos_principal, self.timeout_seconds]))
 
 class ConnectionPooler(object):
   """
@@ -152,6 +154,7 @@ class ConnectionPooler(object):
 
     start_pool_get_time = time.time()
     has_waited_for = 0
+
     while connection is None:
       if get_client_timeout is not None:
         this_round_timeout = max(min(get_client_timeout - has_waited_for, 1), 0)
@@ -181,11 +184,11 @@ class ConnectionPooler(object):
     self.pooldict[_get_pool_key(conf)].put(client)
 
 def _get_pool_key(conf):
-   """
-   Given a ConnectionConfig, return the tuple used as the key in the dictionary
-   of connections by the ConnectionPooler class.
-   """
-   return (conf.klass, conf.host, conf.port)
+  """
+  Given a ConnectionConfig, return the tuple used as the key in the dictionary
+  of connections by the ConnectionPooler class.
+  """
+  return (conf.klass, conf.host, conf.port)
 
 def construct_superclient(conf):
   """
@@ -251,8 +254,8 @@ class PooledClient(object):
 
     # Fetch the thrift client from the pool
     superclient = _connection_pool.get_client(self.conf)
-
     res = getattr(superclient, attr)
+
     if not callable(res):
       # It's a simple attribute. We can put the superclient back in the pool.
       _connection_pool.return_client(self.conf, superclient)
@@ -279,8 +282,7 @@ class PooledClient(object):
                 superclient.transport.open()
 
             superclient.set_timeout(self.conf.timeout_seconds)
-            ret = res(*args, **kwargs)
-            return ret
+            return res(*args, **kwargs)
           except TApplicationException, e:
             # Unknown thrift exception... typically IO errors
             logging.info("Thrift saw an application exception: " + str(e), exc_info=False)
@@ -297,6 +299,7 @@ class PooledClient(object):
             raise
         finally:
           _connection_pool.return_client(self.conf, superclient)
+      wrapper.attr = attr # Save the name of the attribute as it is replaced by 'wrapper'
       return wrapper
 
 

برخی فایل ها در این مقایسه diff نمایش داده نمی شوند زیرا تعداد فایل ها بسیار زیاد است