Jelajahi Sumber

HUE-4726 [core] Fix struct.unpack error for Python 2.6

Jenny Kim 9 tahun lalu
induk
melakukan
26866c5

+ 3 - 3
apps/filebrowser/src/filebrowser/views.py

@@ -771,11 +771,11 @@ def _read_avro(fhandle, path, offset, length, stats):
 
 def _read_parquet(fhandle, path, offset, length, stats):
     try:
-        size = 1 * 1024 * 1024 # Limit readable file, ParquetOptions.limit does not help
-        data = StringIO(fhandle.read(size)) # There is a footer
+        size = 1 * 1024 * 1024  # Limit readable file, ParquetOptions.limit does not help
+        data = StringIO(fhandle.read(size))  # There is a footer
 
         dumped_data = StringIO()
-        parquet._dump(data, ParquetOptions(), out=dumped_data)
+        parquet._dump(data, ParquetOptions(limit=1000), out=dumped_data)
         dumped_data.seek(offset)
         return dumped_data.read()
     except Exception, e:

+ 5 - 5
apps/filebrowser/src/filebrowser/views_test.py

@@ -525,7 +525,7 @@ class TestFileBrowserWithHadoop(object):
     finish = []
     try:
       prefix = self.cluster.fs_prefix + '/test_view_snappy_compressed'
-      self.self.cluster.fs.mkdir(prefix)
+      self.cluster.fs.mkdir(prefix)
 
       f = cluster.fs.open(prefix + '/test-view.snappy', "w")
       f.write(snappy.compress('This is a test of the emergency broadcasting system.'))
@@ -540,22 +540,22 @@ class TestFileBrowserWithHadoop(object):
       f.close()
 
       # Snappy compressed fail
-      response = c.get('/filebrowser/view=%s/test-view.notsnappy?compression=snappy' % prefix)
+      response = self.c.get('/filebrowser/view=%s/test-view.notsnappy?compression=snappy' % prefix)
       assert_true('Failed to decompress' in response.context['message'], response)
 
       # Snappy compressed succeed
-      response = c.get('/filebrowser/view=%s/test-view.snappy' % prefix)
+      response = self.c.get('/filebrowser/view=%s/test-view.snappy' % prefix)
       assert_equal('snappy', response.context['view']['compression'])
       assert_equal(response.context['view']['contents'], 'This is a test of the emergency broadcasting system.', response)
 
       # Snappy compressed succeed
-      response = c.get('/filebrowser/view=%s/test-view.stillsnappy' % prefix)
+      response = self.c.get('/filebrowser/view=%s/test-view.stillsnappy' % prefix)
       assert_equal('snappy', response.context['view']['compression'])
       assert_equal(response.context['view']['contents'], 'The broadcasters of your area in voluntary cooperation with the FCC and other authorities.', response)
 
       # Largest snappy compressed file
       finish.append( MAX_SNAPPY_DECOMPRESSION_SIZE.set_for_testing(1) )
-      response = c.get('/filebrowser/view=%s/test-view.stillsnappy?compression=snappy' % prefix)
+      response = self.c.get('/filebrowser/view=%s/test-view.stillsnappy?compression=snappy' % prefix)
       assert_true('File size is greater than allowed max snappy decompression size of 1' in response.context['message'], response)
 
     finally:

+ 1 - 1
desktop/core/ext-py/parquet-1.1/parquet/__init__.py

@@ -66,7 +66,7 @@ def _check_footer_magic_bytes(fo):
 def _get_footer_size(fo):
     "Readers the footer size in bytes, which is serialized as little endian"
     fo.seek(-8, 2)
-    tup = struct.unpack("<i", fo.read(4))
+    tup = struct.unpack(b"<i", fo.read(4))
     return tup[0]
 
 

+ 9 - 9
desktop/core/ext-py/parquet-1.1/parquet/encoding.py

@@ -31,36 +31,36 @@ def read_plain_int32(fo, count):
     length = 4 * count
     data = fo.read(length)
     if len(data) != length:
-        raise EOFError("Expected {} bytes but got {} bytes".format(length, len(data)))
-    res = struct.unpack("<{}i".format(count), data)
+        raise EOFError("Expected {} bytes but got {0} bytes".format(length, len(data)))
+    res = struct.unpack(b"<{0}i".format(count), data)
     return res
 
 
 def read_plain_int64(fo, count):
     """Reads `count` 64-bit ints using the plain encoding"""
-    return struct.unpack("<{}q".format(count), fo.read(8 * count))
+    return struct.unpack(b"<{0}q".format(count), fo.read(8 * count))
 
 
 def read_plain_int96(fo, count):
     """Reads `count` 96-bit ints using the plain encoding"""
-    items = struct.unpack("<qi" * count, fo.read(12) * count)
+    items = struct.unpack(b"<qi" * count, fo.read(12) * count)
     args = [iter(items)] * 2
     return [q << 32 | i for (q, i) in zip(*args)]
 
 
 def read_plain_float(fo, count):
     """Reads `count` 32-bit floats using the plain encoding"""
-    return struct.unpack("<{}f".format(count), fo.read(4 * count))
+    return struct.unpack(b"<{0}f".format(count), fo.read(4 * count))
 
 
 def read_plain_double(fo, count):
     """Reads `count` 64-bit float (double) using the plain encoding"""
-    return struct.unpack("<{}d".format(count), fo.read(8 * count))
+    return struct.unpack(b"<{0}d".format(count), fo.read(8 * count))
 
 
 def read_plain_byte_array(fo, count):
     """Read `count` byte arrays using the plain encoding"""
-    return [fo.read(struct.unpack("<i", fo.read(4))[0]) for i in range(count)]
+    return [fo.read(struct.unpack(b"<i", fo.read(4))[0]) for i in range(count)]
 
 
 def read_plain_byte_array_fixed(fo, fixed_length):
@@ -92,7 +92,7 @@ def read_unsigned_var_int(fo):
     result = 0
     shift = 0
     while True:
-        byte = struct.unpack("<B", fo.read(1))[0]
+        byte = struct.unpack(b"<B", fo.read(1))[0]
         result |= ((byte & 0x7F) << shift)
         if (byte & 0x80) == 0:
             break
@@ -112,7 +112,7 @@ def read_rle(fo, header, bit_width, debug_logging):
     width = (bit_width + 7) // 8
     data = fo.read(width)
     data = data + zero_data[len(data):]
-    value = struct.unpack("<i", data)[0]
+    value = struct.unpack(b"<i", data)[0]
     if debug_logging:
         logger.debug("Read RLE group with value %s of byte-width %s and count %s",
                      value, width, count)