import array import math import struct import StringIO import logging from ttypes import Type logger = logging.getLogger("parquet") def read_plain_boolean(fo): """Reads a boolean using the plain encoding""" raise NotImplemented def read_plain_int32(fo): """Reads a 32-bit int using the plain encoding""" tup = struct.unpack("> 1 zero_data = "\x00\x00\x00\x00" data = "" width = byte_width(bit_width) if width >= 1: data += fo.read(1) elif width >= 2: data += fo.read(1) elif width >= 3: data += fo.read(1) elif width == 4: data = fo.read(1) data = data + zero_data[len(data):] value = struct.unpack("> 1 logger.debug("Reading a bit-packed run with: %s groups", num_groups) count = num_groups * 8 raw_bytes = array.array('B', fo.read(count)).tolist() current_byte = 0 b = raw_bytes[current_byte] mask = _mask_for_bits(width) bits_in_byte = 8 res = [] while current_byte < len(raw_bytes): # TODO zero-padding could produce extra zero-values logger.debug(" read bitpacked: width=%s bits_in_byte=%s b=%s," " current_byte=%s", width, bits_in_byte, bin(b), current_byte) if bits_in_byte >= width: res.append(b & mask) b >>= width bits_in_byte -= width else: if current_byte + 1 == len(raw_bytes): break # partial results / padding at the end. next_b = raw_bytes[current_byte + 1] borrowed_bits = next_b & _mask_for_bits(width - bits_in_byte) logger.debug(" borrowing %d bits", width - bits_in_byte) logger.debug(" next_b=%s, borrowed_bits=%s", bin(next_b), bin(borrowed_bits)) res.append((borrowed_bits << bits_in_byte) | b) b = next_b >> (width - bits_in_byte) logger.debug(" shifting away: %d", width - bits_in_byte) bits_in_byte = 8 - (width - bits_in_byte) current_byte += 1 logger.debug(" read bitpackage: added: %s", res[-1]) return res def read_bitpacked_deprecated(fo, count, width): res = [] raw_bytes = array.array('B', fo.read(count)).tolist() current_byte = 0 b = raw_bytes[current_byte] mask = _mask_for_bits(width) # TODO implement return res def read_rle_bit_packed_hybrid(fo, width, length=None): """Implemenation of a decoder for the rel/bit-packed hybrid encoding. If length is not specified, then a 32-bit int is read first to grab the length of the encoded data. """ io_obj = fo if length is None: length = read_plain_int32(fo) raw_bytes = fo.read(length) if raw_bytes == '': return None io_obj = StringIO.StringIO(raw_bytes) res = [] while io_obj.tell() < length: header = read_unsigned_var_int(io_obj) if header & 1 == 0: res += read_rle(io_obj, header, width) else: res += read_bitpacked(io_obj, header, width) return res