Przeglądaj źródła

HUE-5009 [core] Backport parquet-python Fix issues invoking struct methods. (#42)

Commit https://github.com/jcrobak/parquet-python/commit/38ee7d5999f371a17371c27f8ed16d7553701cc3
Jenny Kim 9 lat temu
rodzic
commit
0f2fde4

+ 1 - 1
desktop/core/ext-py/parquet-1.1/parquet/__init__.py

@@ -353,7 +353,7 @@ def read_data_page(file_obj, schema_helper, page_header, column_metadata,
 
 
     elif daph.encoding == parquet_thrift.Encoding.PLAIN_DICTIONARY:
     elif daph.encoding == parquet_thrift.Encoding.PLAIN_DICTIONARY:
         # bit_width is stored as single byte.
         # bit_width is stored as single byte.
-        bit_width = struct.unpack("<B", io_obj.read(1))[0]
+        bit_width = struct.unpack(b"<B", io_obj.read(1))[0]
         if debug_logging:
         if debug_logging:
             logger.debug("bit_width: %d", bit_width)
             logger.debug("bit_width: %d", bit_width)
 
 

+ 3 - 3
desktop/core/ext-py/parquet-1.1/parquet/converted_types.py

@@ -53,8 +53,8 @@ def _convert_unsigned(data, fmt):
     """Convert data from signed to unsigned in bulk."""
     """Convert data from signed to unsigned in bulk."""
     num = len(data)
     num = len(data)
     return struct.unpack(
     return struct.unpack(
-        b"{}{}".format(num, fmt.upper()),
-        struct.pack("{}{}".format(num, fmt), *data)
+        b"{0}{0}".format(num, fmt.upper()).encode("utf-8"),
+        struct.pack(b"{0}{0}".format(num, fmt).encode("utf-8"), *data)
     )
     )
 
 
 
 
@@ -89,4 +89,4 @@ def convert_column(data, schemae):
     else:
     else:
         logger.info("Converted type '%s'' not handled",
         logger.info("Converted type '%s'' not handled",
                     parquet_thrift.ConvertedType._VALUES_TO_NAMES[ctype])  # pylint:disable=protected-access
                     parquet_thrift.ConvertedType._VALUES_TO_NAMES[ctype])  # pylint:disable=protected-access
-    return data
+    return data

+ 14 - 9
desktop/core/ext-py/parquet-1.1/parquet/encoding.py

@@ -1,4 +1,5 @@
 """encoding.py - methods for reading parquet encoded data blocks."""
 """encoding.py - methods for reading parquet encoded data blocks."""
+
 from __future__ import absolute_import
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import division
 from __future__ import print_function
 from __future__ import print_function
@@ -10,6 +11,7 @@ import logging
 import math
 import math
 import os
 import os
 import struct
 import struct
+import sys
 
 
 import thriftpy
 import thriftpy
 
 
@@ -18,6 +20,10 @@ parquet_thrift = thriftpy.load(THRIFT_FILE, module_name=str("parquet_thrift"))
 
 
 logger = logging.getLogger("parquet")  # pylint: disable=invalid-name
 logger = logging.getLogger("parquet")  # pylint: disable=invalid-name
 
 
+PY3 = sys.version_info.major > 2
+
+ARRAY_BYTE_STR = u'B' if PY3 else b'B'
+
 
 
 def read_plain_boolean(file_obj, count):
 def read_plain_boolean(file_obj, count):
     """Read `count` booleans using the plain encoding."""
     """Read `count` booleans using the plain encoding."""
@@ -32,14 +38,14 @@ def read_plain_int32(file_obj, count):
     length = 4 * count
     length = 4 * count
     data = file_obj.read(length)
     data = file_obj.read(length)
     if len(data) != length:
     if len(data) != length:
-        raise EOFError("Expected {} bytes but got {0} bytes".format(length, len(data)))
-    res = struct.unpack(b"<{0}i".format(count), data)
+        raise EOFError("Expected {0} bytes but got {1} bytes".format(length, len(data)))
+    res = struct.unpack(b"<{0}i".format(count).encode("utf-8"), data)
     return res
     return res
 
 
 
 
 def read_plain_int64(file_obj, count):
 def read_plain_int64(file_obj, count):
     """Read `count` 64-bit ints using the plain encoding."""
     """Read `count` 64-bit ints using the plain encoding."""
-    return struct.unpack(b"<{0}q".format(count), file_obj.read(8 * count))
+    return struct.unpack(b"<{0}q".format(count).encode("utf-8"), file_obj.read(8 * count))
 
 
 
 
 def read_plain_int96(file_obj, count):
 def read_plain_int96(file_obj, count):
@@ -51,12 +57,12 @@ def read_plain_int96(file_obj, count):
 
 
 def read_plain_float(file_obj, count):
 def read_plain_float(file_obj, count):
     """Read `count` 32-bit floats using the plain encoding."""
     """Read `count` 32-bit floats using the plain encoding."""
-    return struct.unpack(b"<{0}f".format(count), file_obj.read(4 * count))
+    return struct.unpack(b"<{0}f".format(count).encode("utf-8"), file_obj.read(4 * count))
 
 
 
 
 def read_plain_double(file_obj, count):
 def read_plain_double(file_obj, count):
     """Read `count` 64-bit float (double) using the plain encoding."""
     """Read `count` 64-bit float (double) using the plain encoding."""
-    return struct.unpack(b"<{0}d".format(count), file_obj.read(8 * count))
+    return struct.unpack(b"<{0}d".format(count).encode("utf-8"), file_obj.read(8 * count))
 
 
 
 
 def read_plain_byte_array(file_obj, count):
 def read_plain_byte_array(file_obj, count):
@@ -103,8 +109,7 @@ def read_unsigned_var_int(file_obj):
 
 
 
 
 def read_rle(file_obj, header, bit_width, debug_logging):
 def read_rle(file_obj, header, bit_width, debug_logging):
-    """Read a run-length encoded run from the given fo with the given header
-    and bit_width.
+    """Read a run-length encoded run from the given fo with the given header and bit_width.
 
 
     The count is determined from the header and the width is used to grab the
     The count is determined from the header and the width is used to grab the
     value that's repeated. Yields the value repeated count times.
     value that's repeated. Yields the value repeated count times.
@@ -143,7 +148,7 @@ def read_bitpacked(file_obj, header, width, debug_logging):
     if debug_logging:
     if debug_logging:
         logger.debug("Reading a bit-packed run with: %s groups, count %s, bytes %s",
         logger.debug("Reading a bit-packed run with: %s groups, count %s, bytes %s",
                      num_groups, count, byte_count)
                      num_groups, count, byte_count)
-    raw_bytes = array.array(str('B'), file_obj.read(byte_count)).tolist()
+    raw_bytes = array.array(ARRAY_BYTE_STR, file_obj.read(byte_count)).tolist()
     current_byte = 0
     current_byte = 0
     data = raw_bytes[current_byte]
     data = raw_bytes[current_byte]
     mask = _mask_for_bits(width)
     mask = _mask_for_bits(width)
@@ -176,7 +181,7 @@ def read_bitpacked(file_obj, header, width, debug_logging):
 
 
 def read_bitpacked_deprecated(file_obj, byte_count, count, width, debug_logging):
 def read_bitpacked_deprecated(file_obj, byte_count, count, width, debug_logging):
     """Read `count` values from `fo` using the deprecated bitpacking encoding."""
     """Read `count` values from `fo` using the deprecated bitpacking encoding."""
-    raw_bytes = array.array(str('B'), file_obj.read(byte_count)).tolist()
+    raw_bytes = array.array(ARRAY_BYTE_STR, file_obj.read(byte_count)).tolist()
 
 
     mask = _mask_for_bits(width)
     mask = _mask_for_bits(width)
     index = 0
     index = 0

+ 9 - 9
desktop/core/ext-py/parquet-1.1/test/test_encoding.py

@@ -16,28 +16,28 @@ class TestPlain(unittest.TestCase):
         self.assertEqual(
         self.assertEqual(
             999,
             999,
             parquet.encoding.read_plain_int32(
             parquet.encoding.read_plain_int32(
-                io.BytesIO(struct.pack("<i", 999)), 1)[0])
+                io.BytesIO(struct.pack(b"<i", 999)), 1)[0])
 
 
     def test_int64(self):
     def test_int64(self):
         """Test reading bytes containing int64 data."""
         """Test reading bytes containing int64 data."""
         self.assertEqual(
         self.assertEqual(
             999,
             999,
             parquet.encoding.read_plain_int64(
             parquet.encoding.read_plain_int64(
-                io.BytesIO(struct.pack("<q", 999)), 1)[0])
+                io.BytesIO(struct.pack(b"<q", 999)), 1)[0])
 
 
     def test_int96(self):
     def test_int96(self):
         """Test reading bytes containing int96 data."""
         """Test reading bytes containing int96 data."""
         self.assertEqual(
         self.assertEqual(
             999,
             999,
             parquet.encoding.read_plain_int96(
             parquet.encoding.read_plain_int96(
-                io.BytesIO(struct.pack("<qi", 0, 999)), 1)[0])
+                io.BytesIO(struct.pack(b"<qi", 0, 999)), 1)[0])
 
 
     def test_float(self):
     def test_float(self):
         """Test reading bytes containing float data."""
         """Test reading bytes containing float data."""
         self.assertAlmostEquals(
         self.assertAlmostEquals(
             9.99,
             9.99,
             parquet.encoding.read_plain_float(
             parquet.encoding.read_plain_float(
-                io.BytesIO(struct.pack("<f", 9.99)), 1)[0],
+                io.BytesIO(struct.pack(b"<f", 9.99)), 1)[0],
             2)
             2)
 
 
     def test_double(self):
     def test_double(self):
@@ -45,7 +45,7 @@ class TestPlain(unittest.TestCase):
         self.assertEqual(
         self.assertEqual(
             9.99,
             9.99,
             parquet.encoding.read_plain_double(
             parquet.encoding.read_plain_double(
-                io.BytesIO(struct.pack("<d", 9.99)), 1)[0])
+                io.BytesIO(struct.pack(b"<d", 9.99)), 1)[0])
 
 
     def test_fixed(self):
     def test_fixed(self):
         """Test reading bytes containing fixed bytes data."""
         """Test reading bytes containing fixed bytes data."""
@@ -72,7 +72,7 @@ class TestPlain(unittest.TestCase):
     def test_boolean(self):
     def test_boolean(self):
         """Test reading bytes containing boolean data."""
         """Test reading bytes containing boolean data."""
         data = 0b1101
         data = 0b1101
-        fo = io.BytesIO(struct.pack("<i", data))
+        fo = io.BytesIO(struct.pack(b"<i", data))
         self.assertEqual(
         self.assertEqual(
             [True, False, True, True],
             [True, False, True, True],
             parquet.encoding.read_plain_boolean(fo, 1)[:4]
             parquet.encoding.read_plain_boolean(fo, 1)[:4]
@@ -84,7 +84,7 @@ class TestRle(unittest.TestCase):
 
 
     def testFourByteValue(self):
     def testFourByteValue(self):
         """Test reading a run with a single four-byte value."""
         """Test reading a run with a single four-byte value."""
-        fo = io.BytesIO(struct.pack("<i", 1 << 30))
+        fo = io.BytesIO(struct.pack(b"<i", 1 << 30))
         out = parquet.encoding.read_rle(fo, 2 << 1, 30, True)
         out = parquet.encoding.read_rle(fo, 2 << 1, 30, True)
         self.assertEqual([1 << 30] * 2, list(out))
         self.assertEqual([1 << 30] * 2, list(out))
 
 
@@ -94,13 +94,13 @@ class TestVarInt(unittest.TestCase):
 
 
     def testSingleByte(self):
     def testSingleByte(self):
         """Test reading a single byte value."""
         """Test reading a single byte value."""
-        fo = io.BytesIO(struct.pack("<B", 0x7F))
+        fo = io.BytesIO(struct.pack(b"<B", 0x7F))
         out = parquet.encoding.read_unsigned_var_int(fo)
         out = parquet.encoding.read_unsigned_var_int(fo)
         self.assertEqual(0x7F, out)
         self.assertEqual(0x7F, out)
 
 
     def testFourByte(self):
     def testFourByte(self):
         """Test reading a four byte value."""
         """Test reading a four byte value."""
-        fo = io.BytesIO(struct.pack("<BBBB", 0xFF, 0xFF, 0xFF, 0x7F))
+        fo = io.BytesIO(struct.pack(b"<BBBB", 0xFF, 0xFF, 0xFF, 0x7F))
         out = parquet.encoding.read_unsigned_var_int(fo)
         out = parquet.encoding.read_unsigned_var_int(fo)
         self.assertEqual(0x0FFFFFFF, out)
         self.assertEqual(0x0FFFFFFF, out)