فهرست منبع

HUE-2722 [hive] Hive 0.14 can truncate the number of rows

Romain Rigaux 10 سال پیش
والد
کامیت
9580337fc3
2فایلهای تغییر یافته به همراه42 افزوده شده و 1 حذف شده
  1. 4 1
      apps/beeswax/src/beeswax/server/hive_server2_lib.py
  2. 38 0
      apps/beeswax/src/beeswax/tests.py

+ 4 - 1
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -231,7 +231,10 @@ class HiveServerTColumnValue2:
     if bytestring == '' or re.match('^(\x00)+$', bytestring): # HS2 has just \x00 or '', Impala can have \x00\x00...
       return values
     else:
-      return [None if is_null else value for value, is_null in itertools.izip(values, cls.mark_nulls(values, bytestring))]
+      _values = [None if is_null else value for value, is_null in itertools.izip(values, cls.mark_nulls(values, bytestring))]
+      if len(values) != len(_values): # HS2 can have just \x00\x01 instead of \x00\x01\x00...
+        _values.extend(values[len(_values):])
+      return _values
 
 
 class HiveServerDataTable(DataTable):

+ 38 - 0
apps/beeswax/src/beeswax/tests.py

@@ -263,6 +263,44 @@ for x in sys.stdin:
         [u'abc', 1.0, True, 1, u'NULL', u'<a>lala</a>lulu', 'some   spaces'],
       ], content["results"], content)
 
+  def test_result_nullification(self):
+    QUERY = """
+      CREATE TABLE test_result_nullification (a int);
+      INSERT INTO TABLE test_result_nullification
+      VALUES
+      (1), (1), (1), (1), (1), (1), (1), (1),
+      (2), (2), (2), (2), (2), (2), (2), (2),
+      (NULL), (3), (3), (3), (3), (3), (3), (3),
+      (4), (4), (4), (4), (4), (4), (4), (4),
+      (5), (5), (5), (5), (5), (5), (5), (5),
+      (6), (6), (6), (6), (6), (6), (6), (6);
+    """
+    response = _make_query(self.client, QUERY, local=False)
+    content = json.loads(response.content)
+    assert_true('watch_url' in content)
+
+    response = wait_for_query_to_finish(self.client, response, max=180.0)
+    content = fetch_query_result_data(self.client, response)
+
+    QUERY = """
+      SELECT * FROM test_result_nullification;
+    """
+    response = _make_query(self.client, QUERY, local=False)
+    content = json.loads(response.content)
+    assert_true('watch_url' in content)
+
+    response = wait_for_query_to_finish(self.client, response, max=180.0)
+    content = fetch_query_result_data(self.client, response)
+
+    assert_equal([
+        [1], [1], [1], [1], [1], [1], [1], [1],
+        [2], [2], [2], [2], [2], [2], [2], [2],
+        [u'NULL'], [3], [3], [3], [3], [3], [3], [3],
+        [4], [4], [4], [4], [4], [4], [4], [4],
+        [5], [5], [5], [5], [5], [5], [5], [5],
+        [6], [6], [6], [6], [6], [6], [6], [6]
+      ], content["results"], content)
+
   def test_query_with_udf(self):
     """
     Testing query with udf