Преглед на файлове

HUE-1075 [beeswax] Support Hive Server 2

Romain Rigaux преди 12 години
родител
ревизия
1162e67c79

+ 13 - 3
apps/beeswax/gen-py/cli_service/TCLIService-remote → apps/beeswax/gen-py/TCLIService/TCLIService-remote

@@ -1,9 +1,11 @@
 #!/usr/bin/env python
 #
-# Autogenerated by Thrift Compiler (0.7.0)
+# Autogenerated by Thrift Compiler (0.9.0)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
+#  options string: py:new_style
+#
 
 import sys
 import pprint
@@ -18,7 +20,7 @@ from ttypes import *
 
 if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print ''
-  print 'Usage: ' + sys.argv[0] + ' [-h host:port] [-u url] [-f[ramed]] function [arg1 [arg2...]]'
+  print 'Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] function [arg1 [arg2...]]'
   print ''
   print 'Functions:'
   print '  TOpenSessionResp OpenSession(TOpenSessionReq req)'
@@ -37,6 +39,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print '  TCloseOperationResp CloseOperation(TCloseOperationReq req)'
   print '  TGetResultSetMetadataResp GetResultSetMetadata(TGetResultSetMetadataReq req)'
   print '  TFetchResultsResp FetchResults(TFetchResultsReq req)'
+  print '  TGetLogResp GetLog(TGetLogReq req)'
   print ''
   sys.exit(0)
 
@@ -51,7 +54,8 @@ argi = 1
 if sys.argv[argi] == '-h':
   parts = sys.argv[argi+1].split(':')
   host = parts[0]
-  port = int(parts[1])
+  if len(parts) > 1:
+    port = int(parts[1])
   argi += 2
 
 if sys.argv[argi] == '-u':
@@ -183,6 +187,12 @@ elif cmd == 'FetchResults':
     sys.exit(1)
   pp.pprint(client.FetchResults(eval(args[0]),))
 
+elif cmd == 'GetLog':
+  if len(args) != 1:
+    print 'GetLog requires 1 args'
+    sys.exit(1)
+  pp.pprint(client.GetLog(eval(args[0]),))
+
 else:
   print 'Unrecognized method %s' % cmd
   sys.exit(1)

Файловите разлики са ограничени, защото са твърде много
+ 171 - 121
apps/beeswax/gen-py/TCLIService/TCLIService.py


+ 0 - 0
apps/beeswax/gen-py/cli_service/__init__.py → apps/beeswax/gen-py/TCLIService/__init__.py


+ 6 - 2
apps/beeswax/gen-py/cli_service/constants.py → apps/beeswax/gen-py/TCLIService/constants.py

@@ -1,10 +1,12 @@
 #
-# Autogenerated by Thrift Compiler (0.7.0)
+# Autogenerated by Thrift Compiler (0.9.0)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
+#  options string: py:new_style
+#
 
-from thrift.Thrift import *
+from thrift.Thrift import TType, TMessageType, TException, TApplicationException
 from ttypes import *
 
 PRIMITIVE_TYPES = set([
@@ -18,6 +20,7 @@ PRIMITIVE_TYPES = set([
     7,
     8,
     9,
+    15,
 ])
 COMPLEX_TYPES = set([
     10,
@@ -45,4 +48,5 @@ TYPE_NAMES = {
     11 : "MAP",
     12 : "STRUCT",
     13 : "UNIONTYPE",
+    15 : "DECIMAL",
 }

Файловите разлики са ограничени, защото са твърде много
+ 164 - 160
apps/beeswax/gen-py/TCLIService/ttypes.py


+ 1 - 1
apps/beeswax/regenerate_thrift.sh

@@ -22,7 +22,7 @@ cd $(dirname $0)
 
 thrift -I thrift/include -r --gen py:new_style -o ./ thrift/beeswax.thrift
 thrift -I thrift/include -r --gen java:hashcode -o java/src/main thrift/beeswax.thrift
-thrift -I thrift/include -r --gen py:new_style -o ./ thrift/cli_service.thrift
+thrift -I thrift/include -r --gen py:new_style -o ./ thrift/TCLIService.thrift
 
 # We don't need to have generated code for the metastore, since that's
 # in one of the hive jars that we include

+ 1 - 1
apps/beeswax/setup.py

@@ -23,7 +23,7 @@ setup(
       url = 'http://github.com/cloudera/hue',
       description = "Hive Interface on Hue",
       packages = find_packages('src') + find_packages('gen-py'),
-      package_dir = {'hive_metastore': 'gen-py', 'beeswaxd': 'gen-py', 'fb303': 'gen-py', 'cli_service': 'gen-py', '': 'src'},
+      package_dir = {'hive_metastore': 'gen-py', 'beeswaxd': 'gen-py', 'fb303': 'gen-py', 'TCLIService': 'gen-py', '': 'src'},
       install_requires = ['setuptools', 'desktop'],
       entry_points = {
         'desktop.sdk.application': 'beeswax=beeswax',

+ 2 - 3
apps/beeswax/src/beeswax/forms.py

@@ -25,7 +25,6 @@ from desktop.lib.django_forms import ChoiceOrOtherField, MultiForm, SubmitButton
 from filebrowser.forms import PathField
 
 from beeswax import common
-from beeswax.server.dbms import NoSuchObjectException
 from beeswax.models import SavedQuery
 
 
@@ -126,7 +125,7 @@ class SaveResultsForm(DependencyAwareForm):
             self.db.get_table('default', tbl) # Assumes 'default' DB
           self._errors['target_table'] = self.error_class([_('Table already exists')])
           del cleaned_data['target_table']
-        except hive_metastore.ttypes.NoSuchObjectException:
+        except Exception:
           pass
 
     return cleaned_data
@@ -260,7 +259,7 @@ def _clean_tablename(db, name):
     table = db.get_table("default", name)
     if table.name:
       raise forms.ValidationError(_('Table "%(name)s" already exists') % {'name': name})
-  except (hive_metastore.ttypes.NoSuchObjectException, NoSuchObjectException):
+  except Exception:
     return name
 
 

+ 2 - 1
apps/beeswax/src/beeswax/models.py

@@ -32,7 +32,7 @@ from desktop.lib.exceptions_renderable import PopupException
 from beeswax.conf import SERVER_INTERFACE
 from beeswax.design import HQLdesign, hql_query
 from beeswaxd.ttypes import QueryHandle as BeeswaxdQueryHandle, QueryState
-from cli_service.ttypes import TSessionHandle, THandleIdentifier,\
+from TCLIService.ttypes import TSessionHandle, THandleIdentifier,\
   TOperationState, TOperationHandle, TOperationType
 
 
@@ -183,6 +183,7 @@ class HiveServerQueryHistory(QueryHistory):
 
   def save_state(self, new_state):
     self.last_state = new_state.index
+    self.save()
 
 
 class BeeswaxQueryHistory(QueryHistory):

+ 15 - 5
apps/beeswax/src/beeswax/server/dbms.py

@@ -73,7 +73,12 @@ def get_query_server_config(name='beeswax', requires_ddl=False):
   return query_server
 
 
-class QueryServerException: pass
+class QueryServerException(Exception):
+  # Ideally the query handle will be stored here too.
+
+  def __init__(self, e):
+    super(QueryServerException, self).__init__(e)
+
 
 class NoSuchObjectException: pass
 
@@ -332,12 +337,17 @@ class DataTable:
 
 
 # TODO decorator?
-def expand_exception(exc, db):
+def expand_exception(exc, db, handle=None):
   try:
-    log = db.get_log(exc)
-  except:
+    if handle is not None:
+      log = db.get_log(handle)
+    elif hasattr(handle, 'get_rpc_handle'):
+      log = db.get_log(exc)
+    else:
+      log = _("No server logs for this query")
+  except Exception, e:
     # Always show something, even if server has died on the job.
-    log = _("Could not retrieve logs.")
+    log = _("Could not retrieve logs: %s" % e)
 
   if not exc.message:
     error_message = _("Unknown exception.")

+ 91 - 52
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -15,20 +15,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+import logging
 import thrift
 
 from desktop.lib import thrift_util
 
-from cli_service import TCLIService
-from cli_service.ttypes import TOpenSessionReq, TGetTablesReq, TFetchResultsReq,\
+from TCLIService import TCLIService
+from TCLIService.ttypes import TOpenSessionReq, TGetTablesReq, TFetchResultsReq,\
   TStatusCode, TGetResultSetMetadataReq, TGetColumnsReq, TType,\
   TExecuteStatementReq, TGetOperationStatusReq, TFetchOrientation,\
-  TCloseSessionReq, TGetCatalogsReq
+  TCloseSessionReq, TGetSchemasReq, TGetLogReq
 
 from beeswax import conf
 from beeswax.models import Session, HiveServerQueryHandle, HiveServerQueryHistory
-from beeswax.server.dbms import Table, NoSuchObjectException, DataTable
+from beeswax.server.dbms import Table, NoSuchObjectException, DataTable,\
+  QueryServerException
+
+
+LOG = logging.getLogger(__name__)
 
 
 class HiveServerTable(Table):
@@ -235,45 +239,51 @@ class HiveServerClient:
                                           timeout_seconds=conf.BEESWAX_SERVER_CONN_TIMEOUT.get())
 
 
+  def open_session(self, user):
+    req = TOpenSessionReq(username=user.username, configuration={})
+    res = self._client.OpenSession(req)
+
+    sessionId = res.sessionHandle.sessionId
+    LOG.info('Opening session %s' % sessionId)
+
+    encoded_status, encoded_guid = HiveServerQueryHandle(secret=sessionId.secret, guid=sessionId.guid).get()
+
+    return Session.objects.create(owner=user,
+                                  status_code=res.status.statusCode,
+                                  secret=encoded_status,
+                                  guid=encoded_guid,
+                                  server_protocol_version=res.serverProtocolVersion)
+
+
   def call(self, fn, req, status=TStatusCode.SUCCESS_STATUS):
     session = Session.objects.get_session(self.user)
+
     if session is None:
       session = self.open_session(self.user)
+
     if hasattr(req, 'sessionHandle') and req.sessionHandle is None:
       req.sessionHandle = session.get_handle()
 
     res = fn(req)
 
-    if res.status.statusCode == TStatusCode.ERROR_STATUS: #TODO should be TStatusCode.INVALID_HANDLE_STATUS
-      print 'HS2 Session has expired: %s' % res
-      print 'Retrying with a new sessions...'
+    # Not supported currently in HS2: TStatusCode.INVALID_HANDLE_STATUS
+    if res.status.statusCode == TStatusCode.ERROR_STATUS and \
+        res.status.errorMessage is not None and 'Invalid SessionHandle' in res.status.errorMessage:
+      LOG.info('Retrying with a new session because of %s' % res)
 
       session = self.open_session(self.user)
       req.sessionHandle = session.get_handle()
 
-      res = fn(req)
+      # Get back the name of the function to call
+      res = getattr(self._client, fn.attr)(req)
 
     if status is not None and res.status.statusCode not in (
         TStatusCode.SUCCESS_STATUS, TStatusCode.SUCCESS_WITH_INFO_STATUS, TStatusCode.STILL_EXECUTING_STATUS):
-      raise Exception('Bad status for request %s:\n%s' % (req, res))
+      raise QueryServerException(Exception('Bad status for request %s:\n%s' % (req, res)))
     else:
       return res
 
 
-  def open_session(self, user):
-    req = TOpenSessionReq(username=user.username)
-    res = self._client.OpenSession(req)
-
-    sessionId = res.sessionHandle.sessionId
-
-    encoded_status, encoded_guid = HiveServerQueryHandle(secret=sessionId.secret, guid=sessionId.guid).get()
-
-    return Session.objects.create(owner=user,
-                                  status_code=res.status.statusCode,
-                                  secret=encoded_status,
-                                  guid=encoded_guid,
-                                  server_protocol_version=res.serverProtocolVersion)
-
   def close_session(self):
     session = Session.objects.get_session(self.user).get_handle()
 
@@ -282,12 +292,13 @@ class HiveServerClient:
 
 
   def get_databases(self):
-    req = TGetCatalogsReq()
-    res = self.call(self._client.GetCatalogs, req)
+    # GetCatalogs() is not implemented in HS2
+    req = TGetSchemasReq()
+    res = self.call(self._client.GetSchemas, req)
 
     results, schema = self.fetch_result(res.operationHandle)
 
-    return HiveServerTRowSet(results.results, schema.schema).cols(('TABLE_CATALOG',))
+    return HiveServerTRowSet(results.results, schema.schema).cols(('TABLE_SCHEMA',))
 
 
   def get_tables(self, database, table_names):
@@ -300,8 +311,9 @@ class HiveServerClient:
 
 
   def get_table(self, database, table_name):
-    req = TGetTablesReq(schemaName='default', tableName=table_name)
+    req = TGetTablesReq(schemaName=database, tableName=table_name)
     res = self.call(self._client.GetTables, req)
+
     table_results, table_schema = self.fetch_result(res.operationHandle)
 
     # Using 'SELECT * from table' does not show column comments in the metadata
@@ -309,26 +321,31 @@ class HiveServerClient:
     return HiveServerTable(table_results.results, table_schema.schema, desc_results.results, desc_schema.schema)
 
 
-  def execute_query(self, query, max_rows=1000):
-    # TODO: Need jars, UDF etc
-    results, schema = self.execute_statement(statement=query.query['query'], conf_overlay={}, max_rows=max_rows)
+  def execute_query(self, query, max_rows=100):
+    # TODO: Need to set jars, UDF etc
+    self.execute_statement(statement='SET hive.server2.blocking.query=true')
+
+    results, schema = self.execute_statement(statement=query.query['query'], max_rows=max_rows)
     return HiveServerDataTable(results, schema)
 
 
   def execute_async_query(self, query, statement=0):
+    self.execute_statement(statement='SET hive.server2.blocking.query=false')
+
     query_statement = query.get_query_statement(statement)
-    return self.execute_async_statement(statement=query_statement, conf_overlay={})
+    return self.execute_async_statement(statement=query_statement)
 
 
-  def execute_statement(self, statement, conf_overlay=None, max_rows=100):
-    req = TExecuteStatementReq(statement=statement)
+  def execute_statement(self, statement, max_rows=100):
+    req = TExecuteStatementReq(statement=statement, confOverlay={})
     res = self.call(self._client.ExecuteStatement, req)
 
     return self.fetch_result(res.operationHandle, max_rows=max_rows)
 
 
-  def execute_async_statement(self, statement, conf_overlay=None):
-    req = TExecuteStatementReq(statement=statement)
+  def execute_async_statement(self, statement):
+    # confOverlay is not used by Hive Server 2
+    req = TExecuteStatementReq(statement=statement, confOverlay={})
     res = self.call(self._client.ExecuteStatement, req)
 
     return HiveServerQueryHandle(secret=res.operationHandle.operationId.secret,
@@ -338,20 +355,20 @@ class HiveServerClient:
                                  modified_row_count=res.operationHandle.modifiedRowCount)
 
 
-  def fetch_data(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=10000):
-    # The client should check for hasMoreRows and fetch until the result is empty
+  def fetch_data(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=100):
+    # The client should check for hasMoreRows and fetch until the result is empty dues to a HS2 bug
     results, schema = self.fetch_result(operation_handle, orientation, max_rows)
     return HiveServerDataTable(results, schema)
 
 
-  def get_columns(self, table):
-    req = TGetColumnsReq(schemaName='default', tableName=table)
+  def get_columns(self, database, table):
+    req = TGetColumnsReq(schemaName=database, tableName=table)
     res = self.call(self._client.GetColumns, req)
 
     return self.fetch_result(res.operationHandle)
 
 
-  def fetch_result(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=10000):
+  def fetch_result(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=100):
     fetch_req = TFetchResultsReq(operationHandle=operation_handle, orientation=orientation, maxRows=max_rows)
     res = self.call(self._client.FetchResults, fetch_req)
 
@@ -366,9 +383,14 @@ class HiveServerClient:
 
   def get_operation_status(self, operation_handle):
     req = TGetOperationStatusReq(operationHandle=operation_handle)
-    res = self.call(self._client.GetOperationStatus, req)
+    return self.call(self._client.GetOperationStatus, req)
 
-    return res
+
+  def get_log(self, operation_handle):
+    req = TGetLogReq(operationHandle=operation_handle)
+    res = self.call(self._client.GetLog, req)
+
+    return res.log
 
 
 class HiveServerTableCompatible(HiveServerTable):
@@ -387,6 +409,24 @@ class HiveServerTableCompatible(HiveServerTable):
                                     'comment': col.get('comment', ''), }) for col in HiveServerTable.cols.fget(self)]
 
 
+class ResultCompatible:
+
+  def __init__(self, data_table):
+    self.data_table = data_table
+    self.rows = data_table.rows
+    self.has_more = data_table.has_more
+    self.start_row = data_table.startRowOffset
+    self.ready = True
+
+  @property
+  def columns(self):
+    return self.cols()
+
+
+  def cols(self):
+    return [col.name for col in self.data_table.cols()]
+
+
 class HiveServerClientCompatible:
   """Same API as Beeswax"""
 
@@ -402,8 +442,8 @@ class HiveServerClientCompatible:
 
   def get_state(self, handle):
     operationHandle = handle.get_rpc_handle()
-
     res = self._client.get_operation_status(operationHandle)
+
     return HiveServerQueryHistory.STATE_MAP[res.operationState]
 
 
@@ -423,10 +463,8 @@ class HiveServerClientCompatible:
 
     data_table = self._client.fetch_data(operationHandle, orientation=orientation, max_rows=max_rows)
 
-    return type('Result', (object,), {'rows': data_table.rows,
-                                      'columns': [col.name for col in data_table.cols()],
-                                      'has_more': data_table.has_more,
-                                      'start_row': data_table.startRowOffset, })
+    return ResultCompatible(data_table)
+
 
   def dump_config(self):
     return 'Does not exist in HS2'
@@ -436,12 +474,13 @@ class HiveServerClientCompatible:
     return 'Does not exist in HS2'
 
 
-  def get_log(self, *args, **kwargs):
-    return 'No logs retrieval implemented'
+  def get_log(self, handle):
+    operationHandle = handle.get_rpc_handle()
+    return self._client.get_log(operationHandle)
 
 
-  def get_databases(self, database, table_names):
-    return [table['TABLE_CAT'] for table in self._client.get_databases()]
+  def get_databases(self):
+    return [table['TABLE_SCHEMA'] for table in self._client.get_databases()]
 
 
   def get_tables(self, database, table_names):

+ 12 - 15
apps/beeswax/src/beeswax/views.py

@@ -38,7 +38,6 @@ from desktop.lib.exceptions_renderable import PopupException
 
 from hadoop.fs.exceptions import WebHdfsException
 from jobsub.parameterization import find_variables, substitute_variables
-from beeswaxd.ttypes import BeeswaxException, QueryNotFoundException
 
 import beeswax.forms
 import beeswax.design
@@ -323,7 +322,7 @@ def describe_table(request, database, table):
 
   try:
     table_data = db.get_sample(database, table)
-  except BeeswaxException, ex:
+  except Exception, ex:
     error_message, logs = expand_exception(ex, db)
 
   return render("describe_table.mako", request, {
@@ -347,7 +346,7 @@ def drop_table(request, database):
       query_history = db.drop_tables(database, tables_objects, design)
       url = reverse(app_name + ':watch_query', args=[query_history.id]) + '?on_success_url=' + reverse(app_name + ':show_tables')
       return redirect(url)
-    except BeeswaxException, ex:
+    except Exception, ex:
       error_message, log = expand_exception(ex, db)
       error = _("Failed to remove %(tables)s.  Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message}
       raise PopupException(error, title=_("Beeswax Error"), detail=log)
@@ -494,9 +493,7 @@ def execute_query(request, design_id=None):
           else:
             download = request.POST.has_key('download')
             return execute_directly(request, query, query_server, design, on_success_url=on_success_url, download=download)
-        except BeeswaxException, ex:
-          print ex.errorCode
-          print ex.SQLState
+        except Exception, ex:
           error_message, log = expand_exception(ex, db)
   else:
     if design.id is not None:
@@ -562,7 +559,7 @@ def watch_query(request, id):
   if request.method == 'POST' or (not query_history.is_finished() and query_history.is_success() and not query_history.has_results):
     try:
       query_history = db.execute_next_statement(query_history)
-    except BeeswaxException, ex:
+    except Exception, ex:
       pass
 
   # Check query state
@@ -570,7 +567,7 @@ def watch_query(request, id):
   query_history.save_state(state)
 
   if query_history.is_failure():
-    # When we fetch, Beeswax server will throw us a BeeswaxException, which has the
+    # When we fetch, Beeswax server will throw us a Exception, which has the
     # log we want to display.
     return format_preserving_redirect(request, results_url, request.GET)
   elif query_history.is_finished() or (query_history.is_success() and query_history.has_results):
@@ -601,13 +598,13 @@ def watch_query_refresh_json(request, id):
     if not query_history.is_finished() and query_history.is_success() and not query_history.has_results:
       db.execute_next_statement(query_history)
       handle, state = _get_query_handle_and_state(query_history)
-  except BeeswaxException, ex:
+  except Exception, ex:
     LOG.exception(ex)
     handle, state = _get_query_handle_and_state(query_history)
 
   try:
     log = db.get_log(handle)
-  except BeeswaxException, ex:
+  except Exception, ex:
     log = str(ex)
 
   jobs = _parse_out_hadoop_jobs(log)
@@ -682,7 +679,7 @@ def view_results(request, id, first_row=0):
     log = db.get_log(handle)
   except Exception, ex:
     fetch_error = True
-    error_message, log = expand_exception(ex, db)
+    error_message, log = expand_exception(ex, db, handle)
 
   # Handle errors
   error = fetch_error or results is None or expired
@@ -761,7 +758,7 @@ def save_results(request, id):
       try:
         handle, state = _get_query_handle_and_state(query_history)
         result_meta = db.get_results_metadata(handle)
-      except QueryNotFoundException, ex:
+      except Exception, ex:
         LOG.exception(ex)
         raise PopupException(_('Cannot find query.'))
       if result_meta.table_dir:
@@ -787,7 +784,7 @@ def save_results(request, id):
           # To new table
           try:
             return _save_results_ctas(request, query_history, form.cleaned_data['target_table'], result_meta)
-          except BeeswaxException, bex:
+          except Exception, bex:
             LOG.exception(bex)
             error_msg, log = expand_exception(bex, db)
       except WebHdfsException, ex:
@@ -811,7 +808,7 @@ def save_results(request, id):
 def _save_results_ctas(request, query_history, target_table, result_meta):
   """
   Handle saving results as a new table. Returns HTTP response.
-  May raise BeeswaxException, IOError.
+  May raise Exception, IOError.
   """
   query_server = query_history.get_query_server_config() # Query server requires DDL support
   db = dbms.get(request.user)
@@ -1109,7 +1106,7 @@ def _run_parameterized_query(request, design_id, explain):
         return explain_directly(request, query, design, query_server)
       else:
         return execute_directly(request, query, query_server, design)
-    except BeeswaxException, ex:
+    except Exception, ex:
       db = dbms.get(request.user, query_server)
       error_message, log = expand_exception(ex, db)
       return render('execute.mako', request, {

+ 87 - 63
apps/beeswax/thrift/cli_service.thrift → apps/beeswax/thrift/TCLIService.thrift

@@ -16,20 +16,21 @@
 
 // Coding Conventions for this file:
 //
-// Structs and Unions
-// * Struct and union names begin with a "T", and use a capital letter for
-//   each new word, with no underscores.
+// Structs/Enums/Unions
+// * Struct, Enum, and Union names begin with a "T",
+//   and use a capital letter for each new word, with no underscores.
 // * All fields should be declared as either optional or required.
 //
 // Functions
 // * Function names start with a capital letter and have a capital letter for
 //   each new word, with no underscores.
-// * Each function should take exactly one parameter, named FunctionNameReq,
-//   and should return either void or FunctionNameResp. This convention allows
+// * Each function should take exactly one parameter, named TFunctionNameReq,
+//   and should return either void or TFunctionNameResp. This convention allows
 //   incremental updates.
 //
 // Services
-// * Service names should end in the word "Service".
+// * Service names begin with the letter "T", use a capital letter for each
+//   new word (with no underscores), and end with the word "Service".
 
 namespace java org.apache.hive.service.cli.thrift
 namespace cpp apache.hive.service.cli.thrift
@@ -40,7 +41,7 @@ enum TProtocolVersion {
   HIVE_CLI_SERVICE_PROTOCOL_V1
 }
 
-enum TType {
+enum TTypeId {
   BOOLEAN_TYPE,
   TINYINT_TYPE,
   SMALLINT_TYPE,
@@ -55,50 +56,53 @@ enum TType {
   MAP_TYPE,
   STRUCT_TYPE,
   UNION_TYPE,
-  USER_DEFINED_TYPE
-}
-  
-const set<TType> PRIMITIVE_TYPES = [
-  TType.BOOLEAN_TYPE
-  TType.TINYINT_TYPE
-  TType.SMALLINT_TYPE
-  TType.INT_TYPE
-  TType.BIGINT_TYPE
-  TType.FLOAT_TYPE
-  TType.DOUBLE_TYPE
-  TType.STRING_TYPE
-  TType.TIMESTAMP_TYPE
-  TType.BINARY_TYPE
+  USER_DEFINED_TYPE,
+  DECIMAL_TYPE
+}
+
+const set<TTypeId> PRIMITIVE_TYPES = [
+  TTypeId.BOOLEAN_TYPE
+  TTypeId.TINYINT_TYPE
+  TTypeId.SMALLINT_TYPE
+  TTypeId.INT_TYPE
+  TTypeId.BIGINT_TYPE
+  TTypeId.FLOAT_TYPE
+  TTypeId.DOUBLE_TYPE
+  TTypeId.STRING_TYPE
+  TTypeId.TIMESTAMP_TYPE
+  TTypeId.BINARY_TYPE,
+  TTypeId.DECIMAL_TYPE
 ]
 
-const set<TType> COMPLEX_TYPES = [
-  TType.ARRAY_TYPE
-  TType.MAP_TYPE
-  TType.STRUCT_TYPE
-  TType.UNION_TYPE
-  TType.USER_DEFINED_TYPE
+const set<TTypeId> COMPLEX_TYPES = [
+  TTypeId.ARRAY_TYPE
+  TTypeId.MAP_TYPE
+  TTypeId.STRUCT_TYPE
+  TTypeId.UNION_TYPE
+  TTypeId.USER_DEFINED_TYPE
 ]
 
-const set<TType> COLLECTION_TYPES = [
-  TType.ARRAY_TYPE
-  TType.MAP_TYPE
+const set<TTypeId> COLLECTION_TYPES = [
+  TTypeId.ARRAY_TYPE
+  TTypeId.MAP_TYPE
 ]
 
-const map<TType,string> TYPE_NAMES = {
-  TType.BOOLEAN_TYPE: "BOOLEAN",
-  TType.TINYINT_TYPE: "TINYINT",
-  TType.SMALLINT_TYPE: "SMALLINT",
-  TType.INT_TYPE: "INT",
-  TType.BIGINT_TYPE: "BIGINT",
-  TType.FLOAT_TYPE: "FLOAT",
-  TType.DOUBLE_TYPE: "DOUBLE",
-  TType.STRING_TYPE: "STRING",
-  TType.TIMESTAMP_TYPE: "TIMESTAMP",
-  TType.BINARY_TYPE: "BINARY",
-  TType.ARRAY_TYPE: "ARRAY",
-  TType.MAP_TYPE: "MAP",
-  TType.STRUCT_TYPE: "STRUCT",
-  TType.UNION_TYPE: "UNIONTYPE"
+const map<TTypeId,string> TYPE_NAMES = {
+  TTypeId.BOOLEAN_TYPE: "BOOLEAN",
+  TTypeId.TINYINT_TYPE: "TINYINT",
+  TTypeId.SMALLINT_TYPE: "SMALLINT",
+  TTypeId.INT_TYPE: "INT",
+  TTypeId.BIGINT_TYPE: "BIGINT",
+  TTypeId.FLOAT_TYPE: "FLOAT",
+  TTypeId.DOUBLE_TYPE: "DOUBLE",
+  TTypeId.STRING_TYPE: "STRING",
+  TTypeId.TIMESTAMP_TYPE: "TIMESTAMP",
+  TTypeId.BINARY_TYPE: "BINARY",
+  TTypeId.ARRAY_TYPE: "ARRAY",
+  TTypeId.MAP_TYPE: "MAP",
+  TTypeId.STRUCT_TYPE: "STRUCT",
+  TTypeId.UNION_TYPE: "UNIONTYPE"
+  TTypeId.DECIMAL_TYPE: "DECIMAL"
 }
 
 // Thrift does not support recursively defined types or forward declarations,
@@ -150,7 +154,7 @@ typedef i32 TTypeEntryPtr
 struct TPrimitiveTypeEntry {
   // The primitive type token. This must satisfy the condition
   // that type is in the PRIMITIVE_TYPES set.
-  1: required TType type
+  1: required TTypeId type
 }
 
 // Type entry for an ARRAY type.
@@ -204,7 +208,7 @@ struct TColumnDesc {
 
   // The type descriptor for this column
   2: required TTypeDesc typeDesc
-  
+
   // The ordinal position of this column in the schema
   3: required i32 position
 
@@ -254,6 +258,7 @@ struct TDoubleValue {
 }
 
 struct TStringValue {
+  // NULL if value is unset
   1: optional string value
 }
 
@@ -280,7 +285,7 @@ union TColumnValue {
   4: TI32Value    i32Val       // INT
   5: TI64Value    i64Val       // BIGINT, TIMESTAMP
   6: TDoubleValue doubleVal    // FLOAT, DOUBLE
-  7: TStringValue stringVal    // STRING, LIST, MAP, STRUCT, UNIONTYPE, BINARY
+  7: TStringValue stringVal    // STRING, LIST, MAP, STRUCT, UNIONTYPE, BINARY, DECIMAL
 }
 
 // Represents a row in a rowset.
@@ -438,11 +443,11 @@ struct TOperationHandle {
 // OpenSession()
 //
 // Open a session (connection) on the server against
-// which operations may be executed. 
+// which operations may be executed.
 struct TOpenSessionReq {
   // The version of the HiveServer2 protocol that the client is using.
   1: required TProtocolVersion client_protocol = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V1
-  
+
   // Username and password for authentication.
   // Depending on the authentication scheme being used,
   // this information may instead be provided by a lower
@@ -606,13 +611,13 @@ struct TGetTypeInfoReq {
 struct TGetTypeInfoResp {
   1: required TStatus status
   2: optional TOperationHandle operationHandle
-}  
+}
 
 
 // GetCatalogs()
 //
-// Returns the list of catalogs (databases) 
-// Results are ordered by TABLE_CATALOG 
+// Returns the list of catalogs (databases)
+// Results are ordered by TABLE_CATALOG
 //
 // Resultset columns :
 // col1
@@ -633,7 +638,7 @@ struct TGetCatalogsResp {
 
 // GetSchemas()
 //
-// Retrieves the schema names available in this database. 
+// Retrieves the schema names available in this database.
 // The results are ordered by TABLE_CATALOG and TABLE_SCHEM.
 // col1
 // name: TABLE_SCHEM
@@ -722,9 +727,9 @@ struct TGetTablesResp {
 
 // GetTableTypes()
 //
-// Returns the table types available in this database. 
-// The results are ordered by table type. 
-// 
+// Returns the table types available in this database.
+// The results are ordered by table type.
+//
 // col1
 // name: TABLE_TYPE
 // type: STRING
@@ -745,8 +750,8 @@ struct TGetTableTypesResp {
 // Returns a list of columns in the specified tables.
 // The information is returned as a result set which can be fetched
 // using the OperationHandle provided in the response.
-// Results are ordered by TABLE_CAT, TABLE_SCHEM, TABLE_NAME, 
-// and ORDINAL_POSITION. 
+// Results are ordered by TABLE_CAT, TABLE_SCHEM, TABLE_NAME,
+// and ORDINAL_POSITION.
 //
 // Result Set Columns are the same as those for the ODBC CLIColumns
 // function.
@@ -842,7 +847,7 @@ struct TGetFunctionsResp {
   1: required TStatus status
   2: optional TOperationHandle operationHandle
 }
-  
+
 
 // GetOperationStatus()
 //
@@ -937,7 +942,7 @@ struct TFetchResultsReq {
   // The fetch orientation. For V1 this must be either
   // FETCH_NEXT or FETCH_FIRST. Defaults to FETCH_NEXT.
   2: required TFetchOrientation orientation = TFetchOrientation.FETCH_NEXT
-  
+
   // Max number of rows that should be returned in
   // the rowset.
   3: required i64 maxRows
@@ -956,6 +961,22 @@ struct TFetchResultsResp {
   3: optional TRowSet results
 }
 
+// GetLog()
+//
+// Fetch operation log from the server corresponding to
+// a particular OperationHandle.
+struct TGetLogReq {
+  // Operation whose log is requested
+  1: required TOperationHandle operationHandle
+}
+
+struct TGetLogResp {
+  1: required TStatus status
+
+  2: required string log
+
+}
+
 service TCLIService {
 
   TOpenSessionResp OpenSession(1:TOpenSessionReq req);
@@ -981,12 +1002,15 @@ service TCLIService {
   TGetFunctionsResp GetFunctions(1:TGetFunctionsReq req);
 
   TGetOperationStatusResp GetOperationStatus(1:TGetOperationStatusReq req);
-  
+
   TCancelOperationResp CancelOperation(1:TCancelOperationReq req);
 
   TCloseOperationResp CloseOperation(1:TCloseOperationReq req);
 
   TGetResultSetMetadataResp GetResultSetMetadata(1:TGetResultSetMetadataReq req);
-  
+
   TFetchResultsResp FetchResults(1:TFetchResultsReq req);
+
+  TGetLogResp GetLog(1:TGetLogReq req);
 }
+

+ 11 - 8
desktop/core/src/desktop/lib/thrift_util.py

@@ -90,6 +90,8 @@ class ConnectionConfig(object):
     self.kerberos_principal = kerberos_principal
     self.timeout_seconds = timeout_seconds
 
+  def __str__(self):
+    return ', '.join(map(str, [self.klass, self.host, self.port, self.service_name, self.use_sasl, self.kerberos_principal, self.timeout_seconds]))
 
 class ConnectionPooler(object):
   """
@@ -152,6 +154,7 @@ class ConnectionPooler(object):
 
     start_pool_get_time = time.time()
     has_waited_for = 0
+
     while connection is None:
       if get_client_timeout is not None:
         this_round_timeout = max(min(get_client_timeout - has_waited_for, 1), 0)
@@ -181,11 +184,11 @@ class ConnectionPooler(object):
     self.pooldict[_get_pool_key(conf)].put(client)
 
 def _get_pool_key(conf):
-   """
-   Given a ConnectionConfig, return the tuple used as the key in the dictionary
-   of connections by the ConnectionPooler class.
-   """
-   return (conf.klass, conf.host, conf.port)
+  """
+  Given a ConnectionConfig, return the tuple used as the key in the dictionary
+  of connections by the ConnectionPooler class.
+  """
+  return (conf.klass, conf.host, conf.port)
 
 def construct_superclient(conf):
   """
@@ -251,8 +254,8 @@ class PooledClient(object):
 
     # Fetch the thrift client from the pool
     superclient = _connection_pool.get_client(self.conf)
-
     res = getattr(superclient, attr)
+
     if not callable(res):
       # It's a simple attribute. We can put the superclient back in the pool.
       _connection_pool.return_client(self.conf, superclient)
@@ -279,8 +282,7 @@ class PooledClient(object):
                 superclient.transport.open()
 
             superclient.set_timeout(self.conf.timeout_seconds)
-            ret = res(*args, **kwargs)
-            return ret
+            return res(*args, **kwargs)
           except TApplicationException, e:
             # Unknown thrift exception... typically IO errors
             logging.info("Thrift saw an application exception: " + str(e), exc_info=False)
@@ -297,6 +299,7 @@ class PooledClient(object):
             raise
         finally:
           _connection_pool.return_client(self.conf, superclient)
+      wrapper.attr = attr # Save the name of the attribute as it is replaced by 'wrapper'
       return wrapper
 
 

Някои файлове не бяха показани, защото твърде много файлове са промени