Browse Source

[hiveserver2] SparkSql get_tables not returning empty

Romain Rigaux 5 years ago
parent
commit
9e72f2c35f

+ 11 - 6
apps/beeswax/src/beeswax/api.py

@@ -170,7 +170,7 @@ def _autocomplete(db, database=None, table=None, column=None, nested=None, query
     response['code'] = 500
     response['error'] = str(e)
   except Exception as e:
-    LOG.warn('Autocomplete data fetching error: %s' % e)
+    LOG.exception('Autocomplete data fetching error')
     response['code'] = 500
     response['error'] = str(e)
 
@@ -233,10 +233,10 @@ def parameters(request, design_id=None):
     parameterization_form = parameterization_form_cls(prefix="parameterization")
 
     response['parameters'] = [{'parameter': field.html_name, 'name': field.name} for field in parameterization_form]
-    response['status']= 0
+    response['status'] = 0
   else:
     response['parameters'] = []
-    response['status']= 0
+    response['status'] = 0
 
   return JsonResponse(response)
 
@@ -433,7 +433,7 @@ def execute(request, design_id=None):
         'functions': query_form.functions.errors,
       }
   except RuntimeError as e:
-    response['message']= str(e)
+    response['message'] = str(e)
 
   return JsonResponse(response)
 
@@ -670,12 +670,17 @@ def save_results_hive_table(request, query_history_id):
         return JsonResponse(response)
 
       try:
-        query_history = db.create_table_as_a_select(request, query_history, form.target_database, form.cleaned_data['target_table'], result_meta)
+        query_history = db.create_table_as_a_select(
+          request, query_history, form.target_database, form.cleaned_data['target_table'], result_meta
+        )
         response['id'] = query_history.id
         response['query'] = query_history.query
         response['type'] = 'hive-table'
         response['path'] = form.cleaned_data['target_table']
-        response['success_url'] = reverse('metastore:describe_table', kwargs={'database': form.target_database, 'table': form.cleaned_data['target_table']})
+        response['success_url'] = reverse(
+          'metastore:describe_table',
+          kwargs={'database': form.target_database, 'table': form.cleaned_data['target_table']}
+        )
         response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id})
       except Exception as ex:
         error_msg, log = expand_exception(ex, db)

+ 40 - 5
apps/beeswax/src/beeswax/server/dbms.py

@@ -231,7 +231,7 @@ def get_query_server_config(name='beeswax', connector=None):
           'max_number_of_sessions': MAX_NUMBER_OF_SESSIONS.get()
         }
 
-    if name == 'sparksql': # Extends Hive as very similar
+    if name == 'sparksql':  # Extends Hive as very similar
       from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT, USE_SASL as SPARK_USE_SASL
 
       query_server.update({
@@ -314,7 +314,8 @@ class HiveServer2Dbms(object):
   def __init__(self, client, server_type):
     self.client = client
     self.server_type = server_type
-    self.server_name = self.client.query_server['server_name']
+    self.server_name = self.client.query_server.get('dialect') if self.client.query_server['server_name'].isdigit() \
+        else self.client.query_server['server_name']
 
 
   @classmethod
@@ -360,31 +361,65 @@ class HiveServer2Dbms(object):
 
 
   def get_tables_meta(self, database='default', table_names='*', table_types=None):
-    database = database.lower() # Impala is case sensitive
+    database = database.lower()  # Impala is case sensitive
 
     if self.server_name in ('beeswax', 'sparksql'):
       identifier = self.to_matching_wildcard(table_names)
     else:
       identifier = None
-    tables = self.client.get_tables_meta(database, identifier, table_types)
+
+    if self.server_name == 'sparksql':
+      tables = self._get_tables_via_sparksql(database, table_names)
+    else:
+      tables = self.client.get_tables_meta(database, table_names)
+
     if len(tables) <= APPLY_NATURAL_SORT_MAX.get():
       tables = apply_natural_sort(tables, key='name')
     return tables
 
 
   def get_tables(self, database='default', table_names='*', table_types=None):
-    database = database.lower() # Impala is case sensitive
+    database = database.lower()  # Impala is case sensitive
 
     if self.server_name in ('beeswax', 'sparksql'):
       identifier = self.to_matching_wildcard(table_names)
     else:
       identifier = None
+
     tables = self.client.get_tables(database, identifier, table_types)
+
     if len(tables) <= APPLY_NATURAL_SORT_MAX.get():
       tables = apply_natural_sort(tables)
     return tables
 
 
+  def _get_tables_via_sparksql(self, database, table_names='*'):
+    hql = "SHOW TABLES IN %s" % database
+    if table_names != '*':
+      identifier = self.to_matching_wildcard(table_names)
+      hql += " LIKE '%s'" % (identifier)
+
+    query = hql_query(hql)
+    timeout = SERVER_CONN_TIMEOUT.get()
+
+    handle = self.execute_and_wait(query, timeout_sec=timeout)
+
+    if handle:
+      result = self.fetch(handle, rows=5000)
+      self.close(handle)
+
+      # We get back: database | tableName | isTemporary
+      return [{
+          'name': row[1],
+          'type': 'VIEW' if row[2] else 'TABLE',
+          'comment': ''
+        }
+        for row in result.rows()
+      ]
+    else:
+      return []
+
+
   def get_table(self, database, table_name):
     try:
       return self.client.get_table(database, table_name)

+ 13 - 8
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -17,7 +17,6 @@
 
 from builtins import next, filter, map, object
 import logging
-import itertools
 import json
 import re
 import sys
@@ -37,7 +36,7 @@ from desktop.conf import DEFAULT_USER
 from beeswax import conf as beeswax_conf, hive_site
 from beeswax.hive_site import hiveserver2_use_ssl
 from beeswax.conf import CONFIG_WHITELIST, LIST_PARTITIONS_LIMIT
-from beeswax.models import Session, HiveServerQueryHandle, HiveServerQueryHistory, QueryHistory
+from beeswax.models import Session, HiveServerQueryHandle, HiveServerQueryHistory
 from beeswax.server.dbms import Table, DataTable, QueryServerException, InvalidSessionQueryServerException
 
 
@@ -286,7 +285,11 @@ class HiveServerTRow2(object):
 
   def col(self, colName):
     pos = self._get_col_position(colName)
-    return HiveServerTColumnValue2(self.cols[pos]).val[0] # Return only first element
+    try:
+      return HiveServerTColumnValue2(self.cols[pos]).val[0] # Return only first element
+    except:
+      # Bug with SparkSql
+      return ''
 
   def full_col(self, colName):
     pos = self._get_col_position(colName)
@@ -1324,8 +1327,9 @@ class HiveServerTableCompatible(HiveServerTable):
           'name': col.get('col_name', '').strip() if col.get('col_name') else '',
           'type': col.get('data_type', '').strip() if col.get('data_type') else '',
           'comment': col.get('comment', '').strip() if col.get('comment') else ''
-        }) for col in HiveServerTable.cols.fget(self)
-  ]
+        })
+        for col in HiveServerTable.cols.fget(self)
+    ]
 
 
 class ResultCompatible(object):
@@ -1505,9 +1509,10 @@ class HiveServerClientCompatible(object):
     massaged_tables = []
     for table in tables:
       massaged_tables.append({
-        'name': table['TABLE_NAME'],
-        'comment': table['REMARKS'],
-        'type': table['TABLE_TYPE'].capitalize()}
+          'name': table['TABLE_NAME'],
+          'comment': table['REMARKS'],
+          'type': table['TABLE_TYPE'].capitalize()
+        }
       )
     return massaged_tables
 

+ 5 - 4
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -373,10 +373,11 @@ class HS2Api(Api):
         'has_more': results.has_more,
         'data': results.rows(),
         'meta': [{
-          'name': column.name,
-          'type': column.type,
-          'comment': column.comment
-        } for column in results.data_table.cols()],
+            'name': column.name,
+            'type': column.type,
+            'comment': column.comment
+          } for column in results.data_table.cols()
+        ],
         'type': 'table'
     }