import array import math import struct import StringIO import logging from ttypes import Type logger = logging.getLogger("parquet") def read_plain_boolean(fo): """Reads a boolean using the plain encoding""" raise NotImplemented def read_plain_int32(fo): """Reads a 32-bit int using the plain encoding""" tup = struct.unpack("> 1 zero_data = "\x00\x00\x00\x00" data = "" width = byte_width(bit_width) if width >= 1: data += fo.read(1) if width >= 2: data += fo.read(1) if width >= 3: data += fo.read(1) if width == 4: data += fo.read(1) data = data + zero_data[len(data):] value = struct.unpack("> 1 logger.debug("Reading a bit-packed run with: %s groups", num_groups) count = num_groups * 8 raw_bytes = array.array('B', fo.read(count)).tolist() current_byte = 0 b = raw_bytes[current_byte] mask = _mask_for_bits(width) bits_in_byte = 8 res = [] while current_byte < len(raw_bytes): # TODO zero-padding could produce extra zero-values logger.debug(" read bitpacked: width=%s bits_in_byte=%s b=%s," " current_byte=%s", width, bits_in_byte, bin(b), current_byte) if bits_in_byte >= width: res.append(b & mask) b >>= width bits_in_byte -= width else: if current_byte + 1 == len(raw_bytes): break # partial results / padding at the end. next_b = raw_bytes[current_byte + 1] borrowed_bits = next_b & _mask_for_bits(width - bits_in_byte) logger.debug(" borrowing %d bits", width - bits_in_byte) logger.debug(" next_b=%s, borrowed_bits=%s", bin(next_b), bin(borrowed_bits)) res.append((borrowed_bits << bits_in_byte) | b) b = next_b >> (width - bits_in_byte) logger.debug(" shifting away: %d", width - bits_in_byte) bits_in_byte = 8 - (width - bits_in_byte) current_byte += 1 logger.debug(" read bitpackage: added: %s", res[-1]) return res def read_bitpacked_deprecated(fo, byte_count, count, width): raw_bytes = array.array('B', fo.read(byte_count)).tolist() mask = _mask_for_bits(width) index = 0 res = [] word = 0 bits_in_word = 0 while len(res) < count and index <= len(raw_bytes): logger.debug("index = %d", index) logger.debug("bits in word = %d", bits_in_word) logger.debug("word = %s", bin(word)) if bits_in_word >= width: # how many bits over the value is stored offset = (bits_in_word - width) logger.debug("offset = %d", offset) # figure out the value value = (word & (mask << offset)) >> offset logger.debug("value = %d (%s)", value, bin(value)) res.append(value) bits_in_word -= width else: word = (word << 8) | raw_bytes[index] index += 1 bits_in_word += 8 return res def read_rle_bit_packed_hybrid(fo, width, length=None): """Implemenation of a decoder for the rel/bit-packed hybrid encoding. If length is not specified, then a 32-bit int is read first to grab the length of the encoded data. """ io_obj = fo if length is None: length = read_plain_int32(fo) raw_bytes = fo.read(length) if raw_bytes == '': return None io_obj = StringIO.StringIO(raw_bytes) res = [] while io_obj.tell() < length: header = read_unsigned_var_int(io_obj) if header & 1 == 0: res += read_rle(io_obj, header, width) else: res += read_bitpacked(io_obj, header, width) return res