import array import math import struct import cStringIO import logging from ttypes import Type logger = logging.getLogger("parquet") def read_plain_boolean(fo): """Reads a boolean using the plain encoding""" raise NotImplemented def read_plain_int32(fo): """Reads a 32-bit int using the plain encoding""" tup = struct.unpack("> 1 zero_data = "\x00\x00\x00\x00" data = "" width = byte_width(bit_width) if width >= 1: data += fo.read(1) if width >= 2: data += fo.read(1) if width >= 3: data += fo.read(1) if width == 4: data += fo.read(1) data = data + zero_data[len(data):] value = struct.unpack("8 (crossing bytes). """ num_groups = header >> 1 count = num_groups * 8 byte_count = (width * count)/8 logger.debug("Reading a bit-packed run with: %s groups, count %s, bytes %s", num_groups, count, byte_count) raw_bytes = array.array('B', fo.read(byte_count)).tolist() current_byte = 0 b = raw_bytes[current_byte] mask = _mask_for_bits(width) bits_wnd_l = 8 bits_wnd_r = 0 res = [] total = len(raw_bytes)*8; while (total >= width): # TODO zero-padding could produce extra zero-values logger.debug(" read bitpacked: width=%s window=(%s %s) b=%s," " current_byte=%s", width, bits_wnd_l, bits_wnd_r, bin(b), current_byte) if bits_wnd_r >= 8: bits_wnd_r -= 8 bits_wnd_l -= 8 b >>= 8 elif bits_wnd_l - bits_wnd_r >= width: res.append((b >> bits_wnd_r) & mask) total -= width bits_wnd_r += width logger.debug(" read bitpackage: added: %s", res[-1]) elif current_byte + 1 < len(raw_bytes): current_byte += 1 b |= (raw_bytes[current_byte] << bits_wnd_l) bits_wnd_l += 8 return res def read_bitpacked_deprecated(fo, byte_count, count, width): raw_bytes = array.array('B', fo.read(byte_count)).tolist() mask = _mask_for_bits(width) index = 0 res = [] word = 0 bits_in_word = 0 while len(res) < count and index <= len(raw_bytes): logger.debug("index = %d", index) logger.debug("bits in word = %d", bits_in_word) logger.debug("word = %s", bin(word)) if bits_in_word >= width: # how many bits over the value is stored offset = (bits_in_word - width) logger.debug("offset = %d", offset) # figure out the value value = (word & (mask << offset)) >> offset logger.debug("value = %d (%s)", value, bin(value)) res.append(value) bits_in_word -= width else: word = (word << 8) | raw_bytes[index] index += 1 bits_in_word += 8 return res def read_rle_bit_packed_hybrid(fo, width, length=None): """Implemenation of a decoder for the rel/bit-packed hybrid encoding. If length is not specified, then a 32-bit int is read first to grab the length of the encoded data. """ io_obj = fo if length is None: length = read_plain_int32(fo) raw_bytes = fo.read(length) if raw_bytes == '': return None io_obj = cStringIO.StringIO(raw_bytes) res = [] while io_obj.tell() < length: header = read_unsigned_var_int(io_obj) if header & 1 == 0: res += read_rle(io_obj, header, width) else: res += read_bitpacked(io_obj, header, width) return res