Browse Source

HUE-5009 [core] Backport parquet-python Fix issues invoking struct methods. (#42)

Commit https://github.com/jcrobak/parquet-python/commit/38ee7d5999f371a17371c27f8ed16d7553701cc3
Jenny Kim 9 years ago
parent
commit
0f2fde4775

+ 1 - 1
desktop/core/ext-py/parquet-1.1/parquet/__init__.py

@@ -353,7 +353,7 @@ def read_data_page(file_obj, schema_helper, page_header, column_metadata,
 
     elif daph.encoding == parquet_thrift.Encoding.PLAIN_DICTIONARY:
         # bit_width is stored as single byte.
-        bit_width = struct.unpack("<B", io_obj.read(1))[0]
+        bit_width = struct.unpack(b"<B", io_obj.read(1))[0]
         if debug_logging:
             logger.debug("bit_width: %d", bit_width)
 

+ 3 - 3
desktop/core/ext-py/parquet-1.1/parquet/converted_types.py

@@ -53,8 +53,8 @@ def _convert_unsigned(data, fmt):
     """Convert data from signed to unsigned in bulk."""
     num = len(data)
     return struct.unpack(
-        b"{}{}".format(num, fmt.upper()),
-        struct.pack("{}{}".format(num, fmt), *data)
+        b"{0}{0}".format(num, fmt.upper()).encode("utf-8"),
+        struct.pack(b"{0}{0}".format(num, fmt).encode("utf-8"), *data)
     )
 
 
@@ -89,4 +89,4 @@ def convert_column(data, schemae):
     else:
         logger.info("Converted type '%s'' not handled",
                     parquet_thrift.ConvertedType._VALUES_TO_NAMES[ctype])  # pylint:disable=protected-access
-    return data
+    return data

+ 14 - 9
desktop/core/ext-py/parquet-1.1/parquet/encoding.py

@@ -1,4 +1,5 @@
 """encoding.py - methods for reading parquet encoded data blocks."""
+
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -10,6 +11,7 @@ import logging
 import math
 import os
 import struct
+import sys
 
 import thriftpy
 
@@ -18,6 +20,10 @@ parquet_thrift = thriftpy.load(THRIFT_FILE, module_name=str("parquet_thrift"))
 
 logger = logging.getLogger("parquet")  # pylint: disable=invalid-name
 
+PY3 = sys.version_info.major > 2
+
+ARRAY_BYTE_STR = u'B' if PY3 else b'B'
+
 
 def read_plain_boolean(file_obj, count):
     """Read `count` booleans using the plain encoding."""
@@ -32,14 +38,14 @@ def read_plain_int32(file_obj, count):
     length = 4 * count
     data = file_obj.read(length)
     if len(data) != length:
-        raise EOFError("Expected {} bytes but got {0} bytes".format(length, len(data)))
-    res = struct.unpack(b"<{0}i".format(count), data)
+        raise EOFError("Expected {0} bytes but got {1} bytes".format(length, len(data)))
+    res = struct.unpack(b"<{0}i".format(count).encode("utf-8"), data)
     return res
 
 
 def read_plain_int64(file_obj, count):
     """Read `count` 64-bit ints using the plain encoding."""
-    return struct.unpack(b"<{0}q".format(count), file_obj.read(8 * count))
+    return struct.unpack(b"<{0}q".format(count).encode("utf-8"), file_obj.read(8 * count))
 
 
 def read_plain_int96(file_obj, count):
@@ -51,12 +57,12 @@ def read_plain_int96(file_obj, count):
 
 def read_plain_float(file_obj, count):
     """Read `count` 32-bit floats using the plain encoding."""
-    return struct.unpack(b"<{0}f".format(count), file_obj.read(4 * count))
+    return struct.unpack(b"<{0}f".format(count).encode("utf-8"), file_obj.read(4 * count))
 
 
 def read_plain_double(file_obj, count):
     """Read `count` 64-bit float (double) using the plain encoding."""
-    return struct.unpack(b"<{0}d".format(count), file_obj.read(8 * count))
+    return struct.unpack(b"<{0}d".format(count).encode("utf-8"), file_obj.read(8 * count))
 
 
 def read_plain_byte_array(file_obj, count):
@@ -103,8 +109,7 @@ def read_unsigned_var_int(file_obj):
 
 
 def read_rle(file_obj, header, bit_width, debug_logging):
-    """Read a run-length encoded run from the given fo with the given header
-    and bit_width.
+    """Read a run-length encoded run from the given fo with the given header and bit_width.
 
     The count is determined from the header and the width is used to grab the
     value that's repeated. Yields the value repeated count times.
@@ -143,7 +148,7 @@ def read_bitpacked(file_obj, header, width, debug_logging):
     if debug_logging:
         logger.debug("Reading a bit-packed run with: %s groups, count %s, bytes %s",
                      num_groups, count, byte_count)
-    raw_bytes = array.array(str('B'), file_obj.read(byte_count)).tolist()
+    raw_bytes = array.array(ARRAY_BYTE_STR, file_obj.read(byte_count)).tolist()
     current_byte = 0
     data = raw_bytes[current_byte]
     mask = _mask_for_bits(width)
@@ -176,7 +181,7 @@ def read_bitpacked(file_obj, header, width, debug_logging):
 
 def read_bitpacked_deprecated(file_obj, byte_count, count, width, debug_logging):
     """Read `count` values from `fo` using the deprecated bitpacking encoding."""
-    raw_bytes = array.array(str('B'), file_obj.read(byte_count)).tolist()
+    raw_bytes = array.array(ARRAY_BYTE_STR, file_obj.read(byte_count)).tolist()
 
     mask = _mask_for_bits(width)
     index = 0

+ 9 - 9
desktop/core/ext-py/parquet-1.1/test/test_encoding.py

@@ -16,28 +16,28 @@ class TestPlain(unittest.TestCase):
         self.assertEqual(
             999,
             parquet.encoding.read_plain_int32(
-                io.BytesIO(struct.pack("<i", 999)), 1)[0])
+                io.BytesIO(struct.pack(b"<i", 999)), 1)[0])
 
     def test_int64(self):
         """Test reading bytes containing int64 data."""
         self.assertEqual(
             999,
             parquet.encoding.read_plain_int64(
-                io.BytesIO(struct.pack("<q", 999)), 1)[0])
+                io.BytesIO(struct.pack(b"<q", 999)), 1)[0])
 
     def test_int96(self):
         """Test reading bytes containing int96 data."""
         self.assertEqual(
             999,
             parquet.encoding.read_plain_int96(
-                io.BytesIO(struct.pack("<qi", 0, 999)), 1)[0])
+                io.BytesIO(struct.pack(b"<qi", 0, 999)), 1)[0])
 
     def test_float(self):
         """Test reading bytes containing float data."""
         self.assertAlmostEquals(
             9.99,
             parquet.encoding.read_plain_float(
-                io.BytesIO(struct.pack("<f", 9.99)), 1)[0],
+                io.BytesIO(struct.pack(b"<f", 9.99)), 1)[0],
             2)
 
     def test_double(self):
@@ -45,7 +45,7 @@ class TestPlain(unittest.TestCase):
         self.assertEqual(
             9.99,
             parquet.encoding.read_plain_double(
-                io.BytesIO(struct.pack("<d", 9.99)), 1)[0])
+                io.BytesIO(struct.pack(b"<d", 9.99)), 1)[0])
 
     def test_fixed(self):
         """Test reading bytes containing fixed bytes data."""
@@ -72,7 +72,7 @@ class TestPlain(unittest.TestCase):
     def test_boolean(self):
         """Test reading bytes containing boolean data."""
         data = 0b1101
-        fo = io.BytesIO(struct.pack("<i", data))
+        fo = io.BytesIO(struct.pack(b"<i", data))
         self.assertEqual(
             [True, False, True, True],
             parquet.encoding.read_plain_boolean(fo, 1)[:4]
@@ -84,7 +84,7 @@ class TestRle(unittest.TestCase):
 
     def testFourByteValue(self):
         """Test reading a run with a single four-byte value."""
-        fo = io.BytesIO(struct.pack("<i", 1 << 30))
+        fo = io.BytesIO(struct.pack(b"<i", 1 << 30))
         out = parquet.encoding.read_rle(fo, 2 << 1, 30, True)
         self.assertEqual([1 << 30] * 2, list(out))
 
@@ -94,13 +94,13 @@ class TestVarInt(unittest.TestCase):
 
     def testSingleByte(self):
         """Test reading a single byte value."""
-        fo = io.BytesIO(struct.pack("<B", 0x7F))
+        fo = io.BytesIO(struct.pack(b"<B", 0x7F))
         out = parquet.encoding.read_unsigned_var_int(fo)
         self.assertEqual(0x7F, out)
 
     def testFourByte(self):
         """Test reading a four byte value."""
-        fo = io.BytesIO(struct.pack("<BBBB", 0xFF, 0xFF, 0xFF, 0x7F))
+        fo = io.BytesIO(struct.pack(b"<BBBB", 0xFF, 0xFF, 0xFF, 0x7F))
         out = parquet.encoding.read_unsigned_var_int(fo)
         self.assertEqual(0x0FFFFFFF, out)