Browse Source

Initial implementation of BIT_PACK.

Not yet integrated, but passes a simple test.
Joe Crobak 12 years ago
parent
commit
2e12b2823c
2 changed files with 29 additions and 11 deletions
  1. 26 7
      parquet/encoding.py
  2. 3 4
      test/test_encoding.py

+ 26 - 7
parquet/encoding.py

@@ -168,14 +168,33 @@ def read_bitpacked(fo, header, width):
     return res
     return res
 
 
 
 
-def read_bitpacked_deprecated(fo, count, width):
-    res = []
-    raw_bytes = array.array('B', fo.read(count)).tolist()
-    current_byte = 0
-    b = raw_bytes[current_byte]
-    mask = _mask_for_bits(width)
+def read_bitpacked_deprecated(fo, byte_count, count, width):
+    raw_bytes = array.array('B', fo.read(byte_count)).tolist()
 
 
-    # TODO implement
+    mask = _mask_for_bits(width)
+    index = 0
+    res = []
+    word = 0
+    bits_in_word = 0
+    while len(res) < count and index <= len(raw_bytes):
+        logger.debug("index = %d", index)
+        logger.debug("bits in word = %d", bits_in_word)
+        logger.debug("word = %s", bin(word))
+        if bits_in_word >= width:
+            # how many bits over the value is stored
+            offset = (bits_in_word - width)
+            logger.debug("offset = %d", offset)
+
+            # figure out the value
+            value = (word & (mask << offset)) >> offset
+            logger.debug("value = %d (%s)", value, bin(value))
+            res.append(value)
+
+            bits_in_word -= width
+        else:
+            word = (word << 8) | raw_bytes[index]
+            index += 1
+            bits_in_word += 8
     return res
     return res
 
 
 
 

+ 3 - 4
test/test_encoding.py

@@ -97,11 +97,10 @@ class TestBitPacked(unittest.TestCase):
 class TestBitPackedDeprecated(unittest.TestCase):
 class TestBitPackedDeprecated(unittest.TestCase):
 
 
     def testFromExample(self):
     def testFromExample(self):
-        raise SkipTest
-        encoded_bitstring = array.array('B',
-                                        [0b00000101, 0b00111001, 0b01110111])
+        encoded_bitstring = array.array(
+            'B', [0b00000101, 0b00111001, 0b01110111]).tostring()
         fo = StringIO.StringIO(encoded_bitstring)
         fo = StringIO.StringIO(encoded_bitstring)
-        res = parquet.encoding.read_bitpacked_deprecated(fo, 3, 3)
+        res = parquet.encoding.read_bitpacked_deprecated(fo, 3, 8, 3)
         self.assertEquals(range(8), res)
         self.assertEquals(range(8), res)