import array import math import struct import StringIO from ttypes import Type def read_plain_boolean(fo): raise NotImplemented def read_plain_int32(fo): tup = struct.unpack("> 1 zero_data = "\x00\x00\x00\x00" data = "" width = byte_width(bit_width) if width >= 1: data += fo.read(1) elif width >= 2: data += fo.read(1) elif width >= 3: data += fo.read(1) elif width == 4: data = fo.read(1) data = data + zero_data[len(data):] value = struct.unpack("> 1; count = num_groups * 8 raw_bytes = array.array('B', fo.read(count)).tolist() current_byte = 0 b = raw_bytes[current_byte] mask = mask_for_bits(width) bits_in_byte = 8 res = [] while current_byte < width and len(res) < (count / width): print "width={0} bits_in_byte={1} b={2}".format(width, bits_in_byte, bin(b)) if bits_in_byte >= width: res.append(b & mask) b >>= width bits_in_byte -= width else: next_b = raw_bytes[current_byte + 1] borrowed_bits = next_b & mask_for_bits(width - bits_in_byte) #print " borrowing {0} bites".format(width - bits_in_byte) #print " next_b={0}, borrowed_bits={1}".format(bin(next_b), bin(borrowed_bits)) res.append((borrowed_bits << bits_in_byte) | b) b = next_b >> (width - bits_in_byte) #print " shifting away: {0}".format(width - bits_in_byte) bits_in_byte = 8 - (width - bits_in_byte) current_byte += 1 print " added: {0}".format(res[-1]) return res def read_bitpacked_deprecated(fo, count, width): res = [] raw_bytes = array.array('B', fo.read(count)).tolist() current_byte = 0 b = raw_bytes[current_byte] mask = mask_for_bits(width) def read_rle_bit_packed_hybrid(fo, width, length=None): # import pdb; pdb.set_trace() io_obj = fo if length is None: length = read_plain_int32(fo) raw_bytes = fo.read(length) if raw_bytes == '': return None io_obj = StringIO.StringIO(raw_bytes) res = [] while io_obj.tell() < length: header = read_unsigned_var_int(io_obj) if header & 1 == 0: res += read_rle(io_obj, header, width) else: res += read_bitpacked(io_obj, header, width) return res