瀏覽代碼

HUE-2257 [impala] Quicker detection of non null values in a column

Romain Rigaux 10 年之前
父節點
當前提交
d201109

+ 1 - 1
apps/beeswax/src/beeswax/conf.py

@@ -90,7 +90,7 @@ CLOSE_QUERIES = Config(
 
 THRIFT_VERSION = Config(
   key="thrift_version",
-  help=_t("Thrift version to use when communicating with HiveServer2"),
+  help=_t("Thrift version to use when communicating with HiveServer2."),
   type=int,
   default=7
 )

+ 8 - 8
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -57,7 +57,7 @@ class HiveServerTable(Table):
     if beeswax_conf.THRIFT_VERSION.get() >= 7:
       if not table_results.columns:
         raise NoSuchObjectException()
-      self.table = table_results.columns or ''
+      self.table = table_results.columns
     else: # Deprecated. To remove in Hue 4.
       if not table_results.rows:
         raise NoSuchObjectException()
@@ -179,7 +179,7 @@ class HiveServerTRowSet2:
     cols = [rs.full_col(name) for name in col_names]
 
     for cols_row in itertools.izip(*cols):
-      cols_rows.append(dict(zip(col_names, cols_row)))
+      cols_rows.append(dict(itertools.izip(col_names, cols_row)))
 
     return cols_rows
 
@@ -263,10 +263,10 @@ class HiveServerTColumnValue2:
 
   @classmethod
   def set_nulls(cls, values, bytestring):
-    if bytestring == '' or bytestring == '\x00':
+    if bytestring == '' or re.match('^(\x00)+$', bytestring): # HS2 has just \x00 or '', Impala can have \x00\x00...
       return values
     else:
-      return [None if is_null else value for value, is_null in zip(values, cls.mark_nulls(values, bytestring))]
+      return [None if is_null else value for value, is_null in itertools.izip(values, cls.mark_nulls(values, bytestring))]
 
 
 class HiveServerDataTable(DataTable):
@@ -304,7 +304,7 @@ class HiveServerTTableSchema:
   def cols(self):
     try:
       return HiveServerTRowSet(self.columns, self.schema).cols(('col_name', 'data_type', 'comment'))
-    except Exception:
+    except:
       # Impala API is different
       cols = HiveServerTRowSet(self.columns, self.schema).cols(('name', 'type', 'comment'))
       for col in cols:
@@ -627,7 +627,7 @@ class HiveServerClient:
       query = 'DESCRIBE %s' % table_name
     else:
       query = 'DESCRIBE EXTENDED %s' % table_name
-    (desc_results, desc_schema), operation_handle = self.execute_statement(query, max_rows=5000)
+    (desc_results, desc_schema), operation_handle = self.execute_statement(query, max_rows=5000, orientation=TFetchOrientation.FETCH_NEXT)
     self.close_operation(operation_handle)
 
     return HiveServerTable(table_results.results, table_schema.schema, desc_results.results, desc_schema.schema)
@@ -662,14 +662,14 @@ class HiveServerClient:
     return self.execute_async_statement(statement=query_statement, confOverlay=configuration)
 
 
-  def execute_statement(self, statement, max_rows=1000, configuration={}):
+  def execute_statement(self, statement, max_rows=1000, configuration={}, orientation=TFetchOrientation.FETCH_FIRST):
     if self.query_server['server_name'] == 'impala' and self.query_server['QUERY_TIMEOUT_S'] > 0:
       configuration['QUERY_TIMEOUT_S'] = str(self.query_server['QUERY_TIMEOUT_S'])
 
     req = TExecuteStatementReq(statement=statement.encode('utf-8'), confOverlay=configuration)
     res = self.call(self._client.ExecuteStatement, req)
 
-    return self.fetch_result(res.operationHandle, max_rows=max_rows), res.operationHandle
+    return self.fetch_result(res.operationHandle, max_rows=max_rows, orientation=orientation), res.operationHandle
 
 
   def execute_async_statement(self, statement, confOverlay):

+ 17 - 0
apps/beeswax/src/beeswax/tests.py

@@ -2032,6 +2032,23 @@ class TestHiveServer2API():
                   None, 1, 1, 1, None, None, None, 1, None, 1, None, 1, None, None, 1, None, None, 1, 1, None, None, None, None, None, None, 1, None, None, None],
                  HiveServerTColumnValue2.set_nulls(data, nulls))
 
+  def test_column_detect_if_values_nulls(self):
+    data = [1, 2, 3]
+
+    nulls = ''
+    assert_true(data is HiveServerTColumnValue2.set_nulls(data, nulls))
+    nulls = '\x00'
+    assert_true(data is HiveServerTColumnValue2.set_nulls(data, nulls))
+    nulls = '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+    assert_true(data is HiveServerTColumnValue2.set_nulls(data, nulls))
+
+    nulls = 'aaaa'
+    assert_false(data is HiveServerTColumnValue2.set_nulls(data, nulls))
+    nulls = '\x00\x01\x00'
+    assert_false(data is HiveServerTColumnValue2.set_nulls(data, nulls))
+    nulls = '\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00'
+    assert_false(data is HiveServerTColumnValue2.set_nulls(data, nulls))
+
 
 class MockDbms:
 

+ 4 - 1
apps/metastore/src/metastore/views.py

@@ -146,7 +146,10 @@ def describe_table(request, database, table):
   try:
     table = db.get_table(database, table)
   except Exception, e:
-    raise PopupException(_("Hive Error"), detail=e)
+    if hasattr(e, 'message') and e.message:
+      raise PopupException(_("Hive Error"), detail=e.message)
+    else:
+      raise PopupException(_("Hive Error"), detail=e)
 
   partitions = None
   if app_name != 'impala' and table.partition_keys:

+ 1 - 0
desktop/conf.dist/hue.ini

@@ -793,6 +793,7 @@
   ## close_queries=false
 
   # Thrift version to use when communicating with HiveServer2.
+  # New column format is from version 7.
   ## thrift_version=7
 
   [[ssl]]