소스 검색

[beeswax] adding config flag for max no of objects in the left assist, autocomplete, table browser etc.

ayush.goyal 4 년 전
부모
커밋
c01df1ce5b
4개의 변경된 파일31개의 추가작업 그리고 7개의 파일을 삭제
  1. 9 0
      apps/beeswax/src/beeswax/conf.py
  2. 14 7
      apps/beeswax/src/beeswax/server/hive_server2_lib.py
  3. 4 0
      desktop/conf.dist/hue.ini
  4. 4 0
      desktop/conf/pseudo-distributed.ini.tmpl

+ 9 - 0
apps/beeswax/src/beeswax/conf.py

@@ -358,3 +358,12 @@ CLOSE_SESSIONS = Config(
 
 def has_session_pool():
   return has_multiple_sessions() and not CLOSE_SESSIONS.get()
+
+MAX_CATALOG_SQL_ENTRIES = Config(
+  key="max_catalog_sql_entries",
+  help=_t(
+    "Max number of objects (columns, tables, databases) available to list in the left assist, autocomplete, table browser etc."
+    "Setting this higher than the default can degrade performance."),
+  default=5000,
+  type=int
+)

+ 14 - 7
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -34,7 +34,7 @@ from desktop.conf import DEFAULT_USER, USE_THRIFT_HTTP_JWT
 
 from beeswax import conf as beeswax_conf, hive_site
 from beeswax.hive_site import hiveserver2_use_ssl
-from beeswax.conf import CONFIG_WHITELIST, LIST_PARTITIONS_LIMIT, HPLSQL
+from beeswax.conf import CONFIG_WHITELIST, LIST_PARTITIONS_LIMIT, HPLSQL, MAX_CATALOG_SQL_ENTRIES
 from beeswax.models import Session, HiveServerQueryHandle, HiveServerQueryHistory
 from beeswax.server.dbms import Table, DataTable, QueryServerException, InvalidSessionQueryServerException
 
@@ -179,7 +179,7 @@ class HiveServerTable(Table):
     try:
       end_cols_index = list(map(itemgetter('col_name'), rows[col_row_index:])).index('')
     except ValueError as e:
-      end_cols_index = 5000
+      end_cols_index = MAX_CATALOG_SQL_ENTRIES.get()
       LOG.warning('Could not guess end column index, so defaulting to %s: %s' % (end_cols_index, e))
     return [{
           'col_name': prop['col_name'].strip() if prop['col_name'] else prop['col_name'],
@@ -822,7 +822,9 @@ class HiveServerClient(object):
 
     (res, session) = self.call(self._client.GetSchemas, req)
 
-    results, schema = self.fetch_result(res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=5000)
+    results, schema = self.fetch_result(
+      res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=MAX_CATALOG_SQL_ENTRIES.get()
+    )
     self._close(res.operationHandle, session)
 
     col = 'TABLE_SCHEM'
@@ -832,8 +834,9 @@ class HiveServerClient(object):
   def get_database(self, database):
     query = 'DESCRIBE DATABASE EXTENDED `%s`' % (database)
 
-    desc_results, desc_schema, operation_handle, session = self.execute_statement(query, max_rows=5000,
-                                                                                  orientation=TFetchOrientation.FETCH_NEXT)
+    desc_results, desc_schema, operation_handle, session = self.execute_statement(
+      query, max_rows=MAX_CATALOG_SQL_ENTRIES.get(), orientation=TFetchOrientation.FETCH_NEXT
+    )
     self._close(operation_handle, session)
 
     if self.query_server.get('dialect') == 'impala':
@@ -856,7 +859,9 @@ class HiveServerClient(object):
     req = TGetTablesReq(schemaName=database, tableName=table_names, tableTypes=table_types)
     (res, session) = self.call(self._client.GetTables, req)
 
-    results, schema = self.fetch_result(res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=5000)
+    results, schema = self.fetch_result(
+      res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=MAX_CATALOG_SQL_ENTRIES.get()
+    )
     self._close(res.operationHandle, session)
 
     cols = ('TABLE_NAME', 'TABLE_TYPE', 'REMARKS')
@@ -869,7 +874,9 @@ class HiveServerClient(object):
     req = TGetTablesReq(schemaName=database, tableName=table_names, tableTypes=table_types)
     (res, session) = self.call(self._client.GetTables, req)
 
-    results, schema = self.fetch_result(res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=5000)
+    results, schema = self.fetch_result(
+      res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=MAX_CATALOG_SQL_ENTRIES.get()
+    )
     self._close(res.operationHandle, session)
 
     return HiveServerTRowSet(results.results, schema.schema).cols(('TABLE_NAME',))

+ 4 - 0
desktop/conf.dist/hue.ini

@@ -1386,6 +1386,10 @@ submit_to=True
 # Enable the HPLSQL mode.
 ## hplsql=false
 
+# Max number of objects (columns, tables, databases) available to list in the left assist, autocomplete, table browser etc.
+# Setting this higher than the default can degrade performance.
+## max_catalog_sql_entries=5000
+
 [[ssl]]
 # Path to Certificate Authority certificates.
 ## cacerts=/etc/hue/cacerts.pem

+ 4 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1369,6 +1369,10 @@
   # Enable the HPLSQL mode.
   ## hplsql=false
 
+  # Max number of objects (columns, tables, databases) available to list in the left assist, autocomplete, table browser etc.
+  # Setting this higher than the default can degrade performance.
+  ## max_catalog_sql_entries=5000
+
   [[ssl]]
     # Path to Certificate Authority certificates.
     ## cacerts=/etc/hue/cacerts.pem